2024-12-10 12:33:22,902 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 2024-12-10 12:33:22,912 main DEBUG Took 0.008422 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 12:33:22,912 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 12:33:22,913 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 12:33:22,913 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 12:33:22,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,920 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 12:33:22,930 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,931 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,931 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,932 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,932 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,932 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,933 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,933 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,933 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,934 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,934 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,934 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,935 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,935 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,935 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,935 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,936 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,936 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,936 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,937 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,937 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,937 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,937 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:33:22,938 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,938 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 12:33:22,940 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:33:22,941 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 12:33:22,942 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 12:33:22,943 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 12:33:22,944 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 12:33:22,944 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 12:33:22,951 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 12:33:22,953 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 12:33:22,954 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 12:33:22,954 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 12:33:22,955 main DEBUG createAppenders(={Console}) 2024-12-10 12:33:22,955 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 initialized 2024-12-10 12:33:22,956 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 2024-12-10 12:33:22,956 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 OK. 2024-12-10 12:33:22,956 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 12:33:22,957 main DEBUG OutputStream closed 2024-12-10 12:33:22,957 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 12:33:22,957 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 12:33:22,957 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@54e1c68b OK 2024-12-10 12:33:23,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 12:33:23,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 12:33:23,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 12:33:23,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 12:33:23,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 12:33:23,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 12:33:23,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 12:33:23,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 12:33:23,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 12:33:23,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 12:33:23,023 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 12:33:23,023 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 12:33:23,023 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 12:33:23,023 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 12:33:23,024 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 12:33:23,024 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 12:33:23,024 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 12:33:23,025 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 12:33:23,027 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 12:33:23,027 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@7dda48d9) with optional ClassLoader: null 2024-12-10 12:33:23,027 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 12:33:23,028 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@7dda48d9] started OK. 2024-12-10T12:33:23,208 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5 2024-12-10 12:33:23,211 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 12:33:23,211 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T12:33:23,219 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithAdaptivePolicy timeout: 13 mins 2024-12-10T12:33:23,236 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-10T12:33:23,238 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/cluster_c0fdc2cf-aabc-c2dd-a875-8dc8dd174cbc, deleteOnExit=true 2024-12-10T12:33:23,238 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-10T12:33:23,239 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/test.cache.data in system properties and HBase conf 2024-12-10T12:33:23,240 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/hadoop.tmp.dir in system properties and HBase conf 2024-12-10T12:33:23,240 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/hadoop.log.dir in system properties and HBase conf 2024-12-10T12:33:23,240 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-10T12:33:23,241 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-10T12:33:23,241 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-10T12:33:23,324 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T12:33:23,401 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:33:23,405 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-10T12:33:23,405 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-10T12:33:23,405 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-10T12:33:23,406 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T12:33:23,406 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-10T12:33:23,407 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-10T12:33:23,407 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T12:33:23,407 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T12:33:23,408 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-10T12:33:23,408 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/nfs.dump.dir in system properties and HBase conf 2024-12-10T12:33:23,408 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/java.io.tmpdir in system properties and HBase conf 2024-12-10T12:33:23,409 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T12:33:23,409 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-10T12:33:23,409 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-10T12:33:24,330 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-10T12:33:24,396 INFO [Time-limited test {}] log.Log(170): Logging initialized @2104ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-10T12:33:24,459 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T12:33:24,516 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T12:33:24,539 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T12:33:24,540 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T12:33:24,542 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T12:33:24,555 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T12:33:24,557 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8167a4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/hadoop.log.dir/,AVAILABLE} 2024-12-10T12:33:24,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62a9beb2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T12:33:24,720 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b03fcff{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/java.io.tmpdir/jetty-localhost-38155-hadoop-hdfs-3_4_1-tests_jar-_-any-2636036415517719998/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T12:33:24,726 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@e0a3ea0{HTTP/1.1, (http/1.1)}{localhost:38155} 2024-12-10T12:33:24,726 INFO [Time-limited test {}] server.Server(415): Started @2435ms 2024-12-10T12:33:25,189 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T12:33:25,195 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T12:33:25,196 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T12:33:25,196 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T12:33:25,196 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T12:33:25,197 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@134e7cc5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/hadoop.log.dir/,AVAILABLE} 2024-12-10T12:33:25,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ca71a25{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T12:33:25,290 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10ba49e9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/java.io.tmpdir/jetty-localhost-43425-hadoop-hdfs-3_4_1-tests_jar-_-any-17522830036179568830/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T12:33:25,291 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@661c2e9c{HTTP/1.1, (http/1.1)}{localhost:43425} 2024-12-10T12:33:25,291 INFO [Time-limited test {}] server.Server(415): Started @3000ms 2024-12-10T12:33:25,337 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T12:33:26,010 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/cluster_c0fdc2cf-aabc-c2dd-a875-8dc8dd174cbc/dfs/data/data2/current/BP-2145257419-172.17.0.2-1733834003915/current, will proceed with Du for space computation calculation, 2024-12-10T12:33:26,010 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/cluster_c0fdc2cf-aabc-c2dd-a875-8dc8dd174cbc/dfs/data/data1/current/BP-2145257419-172.17.0.2-1733834003915/current, will proceed with Du for space computation calculation, 2024-12-10T12:33:26,035 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T12:33:26,080 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a266bae938a2858 with lease ID 0x94e1c5b7aa85150e: Processing first storage report for DS-0b8b429d-7aad-4484-b346-eb8afedaf46b from datanode DatanodeRegistration(127.0.0.1:33875, datanodeUuid=34eaf696-2436-4d2b-85ae-880ce4bf3216, infoPort=46143, infoSecurePort=0, ipcPort=36701, storageInfo=lv=-57;cid=testClusterID;nsid=1276422940;c=1733834003915) 2024-12-10T12:33:26,081 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a266bae938a2858 with lease ID 0x94e1c5b7aa85150e: from storage DS-0b8b429d-7aad-4484-b346-eb8afedaf46b node DatanodeRegistration(127.0.0.1:33875, datanodeUuid=34eaf696-2436-4d2b-85ae-880ce4bf3216, infoPort=46143, infoSecurePort=0, ipcPort=36701, storageInfo=lv=-57;cid=testClusterID;nsid=1276422940;c=1733834003915), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-10T12:33:26,081 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a266bae938a2858 with lease ID 0x94e1c5b7aa85150e: Processing first storage report for DS-849ab310-d8bd-4711-ac6e-98a2b128a961 from datanode DatanodeRegistration(127.0.0.1:33875, datanodeUuid=34eaf696-2436-4d2b-85ae-880ce4bf3216, infoPort=46143, infoSecurePort=0, ipcPort=36701, storageInfo=lv=-57;cid=testClusterID;nsid=1276422940;c=1733834003915) 2024-12-10T12:33:26,081 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a266bae938a2858 with lease ID 0x94e1c5b7aa85150e: from storage DS-849ab310-d8bd-4711-ac6e-98a2b128a961 node DatanodeRegistration(127.0.0.1:33875, datanodeUuid=34eaf696-2436-4d2b-85ae-880ce4bf3216, infoPort=46143, infoSecurePort=0, ipcPort=36701, storageInfo=lv=-57;cid=testClusterID;nsid=1276422940;c=1733834003915), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-10T12:33:26,116 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5 2024-12-10T12:33:26,175 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/cluster_c0fdc2cf-aabc-c2dd-a875-8dc8dd174cbc/zookeeper_0, clientPort=64063, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/cluster_c0fdc2cf-aabc-c2dd-a875-8dc8dd174cbc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/cluster_c0fdc2cf-aabc-c2dd-a875-8dc8dd174cbc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T12:33:26,184 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=64063 2024-12-10T12:33:26,192 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T12:33:26,194 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T12:33:26,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741825_1001 (size=7) 2024-12-10T12:33:26,788 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 with version=8 2024-12-10T12:33:26,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/hbase-staging 2024-12-10T12:33:26,891 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T12:33:27,127 INFO [Time-limited test {}] client.ConnectionUtils(129): master/fad905551524:0 server-side Connection retries=45 2024-12-10T12:33:27,142 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:33:27,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T12:33:27,143 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T12:33:27,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:33:27,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T12:33:27,255 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T12:33:27,303 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T12:33:27,311 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T12:33:27,314 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T12:33:27,337 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 41236 (auto-detected) 2024-12-10T12:33:27,338 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-10T12:33:27,355 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45051 2024-12-10T12:33:27,362 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T12:33:27,365 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T12:33:27,376 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:45051 connecting to ZooKeeper ensemble=127.0.0.1:64063 2024-12-10T12:33:27,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:450510x0, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T12:33:27,467 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45051-0x1000fee7b100000 connected 2024-12-10T12:33:27,542 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T12:33:27,546 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:33:27,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T12:33:27,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45051 2024-12-10T12:33:27,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45051 2024-12-10T12:33:27,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45051 2024-12-10T12:33:27,556 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45051 2024-12-10T12:33:27,556 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45051 2024-12-10T12:33:27,563 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3, hbase.cluster.distributed=false 2024-12-10T12:33:27,617 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/fad905551524:0 server-side Connection retries=45 2024-12-10T12:33:27,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:33:27,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T12:33:27,618 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T12:33:27,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:33:27,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T12:33:27,620 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T12:33:27,622 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T12:33:27,623 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38809 2024-12-10T12:33:27,624 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T12:33:27,629 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T12:33:27,631 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T12:33:27,633 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T12:33:27,636 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:38809 connecting to ZooKeeper ensemble=127.0.0.1:64063 2024-12-10T12:33:27,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388090x0, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T12:33:27,647 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388090x0, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T12:33:27,647 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38809-0x1000fee7b100001 connected 2024-12-10T12:33:27,649 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:33:27,650 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T12:33:27,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38809 2024-12-10T12:33:27,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38809 2024-12-10T12:33:27,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38809 2024-12-10T12:33:27,653 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38809 2024-12-10T12:33:27,653 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38809 2024-12-10T12:33:27,655 INFO [master/fad905551524:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/fad905551524,45051,1733834006886 2024-12-10T12:33:27,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:33:27,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:33:27,670 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fad905551524,45051,1733834006886 2024-12-10T12:33:27,671 DEBUG [M:0;fad905551524:45051 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fad905551524:45051 2024-12-10T12:33:27,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T12:33:27,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T12:33:27,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:27,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:27,697 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T12:33:27,698 INFO [master/fad905551524:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fad905551524,45051,1733834006886 from backup master directory 2024-12-10T12:33:27,698 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T12:33:27,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:33:27,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fad905551524,45051,1733834006886 2024-12-10T12:33:27,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:33:27,711 WARN [master/fad905551524:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T12:33:27,711 INFO [master/fad905551524:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fad905551524,45051,1733834006886 2024-12-10T12:33:27,713 INFO [master/fad905551524:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T12:33:27,715 INFO [master/fad905551524:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T12:33:27,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741826_1002 (size=42) 2024-12-10T12:33:28,181 DEBUG [master/fad905551524:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/hbase.id with ID: 4106563a-8ee9-4f65-b460-5396dace9b65 2024-12-10T12:33:28,224 INFO [master/fad905551524:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T12:33:28,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:28,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:28,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741827_1003 (size=196) 2024-12-10T12:33:28,722 INFO [master/fad905551524:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:33:28,724 INFO [master/fad905551524:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T12:33:28,738 DEBUG [master/fad905551524:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:28,742 INFO [master/fad905551524:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:33:28,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741828_1004 (size=1189) 2024-12-10T12:33:29,183 INFO [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store 2024-12-10T12:33:29,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741829_1005 (size=34) 2024-12-10T12:33:29,200 INFO [master/fad905551524:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T12:33:29,200 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:29,201 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T12:33:29,201 INFO [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:33:29,201 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:33:29,202 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T12:33:29,202 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:33:29,202 INFO [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:33:29,202 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-10T12:33:29,204 WARN [master/fad905551524:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/.initializing 2024-12-10T12:33:29,204 DEBUG [master/fad905551524:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/WALs/fad905551524,45051,1733834006886 2024-12-10T12:33:29,210 INFO [master/fad905551524:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T12:33:29,220 INFO [master/fad905551524:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fad905551524%2C45051%2C1733834006886, suffix=, logDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/WALs/fad905551524,45051,1733834006886, archiveDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/oldWALs, maxLogs=10 2024-12-10T12:33:29,238 DEBUG [master/fad905551524:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/WALs/fad905551524,45051,1733834006886/fad905551524%2C45051%2C1733834006886.1733834009224, exclude list is [], retry=0 2024-12-10T12:33:29,252 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33875,DS-0b8b429d-7aad-4484-b346-eb8afedaf46b,DISK] 2024-12-10T12:33:29,254 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-10T12:33:29,284 INFO [master/fad905551524:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/WALs/fad905551524,45051,1733834006886/fad905551524%2C45051%2C1733834006886.1733834009224 2024-12-10T12:33:29,284 DEBUG [master/fad905551524:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46143:46143)] 2024-12-10T12:33:29,285 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:33:29,285 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:29,288 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:33:29,289 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:33:29,320 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:33:29,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T12:33:29,341 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:29,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:33:29,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:33:29,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T12:33:29,348 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:29,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:29,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:33:29,353 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T12:33:29,353 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:29,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:29,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:33:29,357 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T12:33:29,357 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:29,358 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:29,361 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:33:29,362 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:33:29,370 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T12:33:29,374 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:33:29,378 DEBUG [master/fad905551524:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:33:29,379 INFO [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74319374, jitterRate=0.10744497179985046}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T12:33:29,383 DEBUG [master/fad905551524:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-10T12:33:29,384 INFO [master/fad905551524:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T12:33:29,408 DEBUG [master/fad905551524:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45749f29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:29,435 INFO [master/fad905551524:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-10T12:33:29,444 INFO [master/fad905551524:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T12:33:29,445 INFO [master/fad905551524:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T12:33:29,446 INFO [master/fad905551524:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T12:33:29,448 INFO [master/fad905551524:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-12-10T12:33:29,452 INFO [master/fad905551524:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-12-10T12:33:29,452 INFO [master/fad905551524:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T12:33:29,473 INFO [master/fad905551524:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T12:33:29,483 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T12:33:29,526 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-10T12:33:29,529 INFO [master/fad905551524:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T12:33:29,531 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T12:33:29,538 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-10T12:33:29,541 INFO [master/fad905551524:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T12:33:29,546 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T12:33:29,554 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-10T12:33:29,556 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T12:33:29,563 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T12:33:29,575 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T12:33:29,584 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T12:33:29,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T12:33:29,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T12:33:29,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:29,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:29,597 INFO [master/fad905551524:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=fad905551524,45051,1733834006886, sessionid=0x1000fee7b100000, setting cluster-up flag (Was=false) 2024-12-10T12:33:29,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:29,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:29,655 DEBUG [master/fad905551524:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T12:33:29,658 DEBUG [master/fad905551524:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fad905551524,45051,1733834006886 2024-12-10T12:33:29,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:29,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:29,704 DEBUG [master/fad905551524:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T12:33:29,706 DEBUG [master/fad905551524:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fad905551524,45051,1733834006886 2024-12-10T12:33:29,768 DEBUG [RS:0;fad905551524:38809 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fad905551524:38809 2024-12-10T12:33:29,770 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1008): ClusterId : 4106563a-8ee9-4f65-b460-5396dace9b65 2024-12-10T12:33:29,772 DEBUG [RS:0;fad905551524:38809 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T12:33:29,782 DEBUG [RS:0;fad905551524:38809 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T12:33:29,782 DEBUG [RS:0;fad905551524:38809 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T12:33:29,794 DEBUG [RS:0;fad905551524:38809 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T12:33:29,794 DEBUG [master/fad905551524:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-10T12:33:29,795 DEBUG [RS:0;fad905551524:38809 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431497e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:29,796 DEBUG [RS:0;fad905551524:38809 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a87bf68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fad905551524/172.17.0.2:0 2024-12-10T12:33:29,799 INFO [RS:0;fad905551524:38809 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-10T12:33:29,799 INFO [master/fad905551524:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-10T12:33:29,799 INFO [RS:0;fad905551524:38809 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-10T12:33:29,799 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-10T12:33:29,801 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(3073): reportForDuty to master=fad905551524,45051,1733834006886 with isa=fad905551524/172.17.0.2:38809, startcode=1733834007617 2024-12-10T12:33:29,801 INFO [master/fad905551524:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T12:33:29,806 DEBUG [master/fad905551524:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fad905551524,45051,1733834006886 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T12:33:29,809 DEBUG [master/fad905551524:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fad905551524:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:33:29,809 DEBUG [master/fad905551524:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fad905551524:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:33:29,809 DEBUG [master/fad905551524:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fad905551524:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:33:29,810 DEBUG [master/fad905551524:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fad905551524:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:33:29,810 DEBUG [master/fad905551524:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fad905551524:0, corePoolSize=10, maxPoolSize=10 2024-12-10T12:33:29,810 DEBUG [master/fad905551524:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,810 DEBUG [master/fad905551524:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fad905551524:0, corePoolSize=2, maxPoolSize=2 2024-12-10T12:33:29,810 DEBUG [master/fad905551524:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,811 INFO [master/fad905551524:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733834039811 2024-12-10T12:33:29,812 DEBUG [RS:0;fad905551524:38809 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T12:33:29,813 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T12:33:29,814 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T12:33:29,815 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-10T12:33:29,815 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-10T12:33:29,817 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T12:33:29,817 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T12:33:29,818 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T12:33:29,818 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T12:33:29,818 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,819 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:29,819 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T12:33:29,819 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T12:33:29,820 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T12:33:29,820 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T12:33:29,822 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T12:33:29,822 INFO [master/fad905551524:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T12:33:29,824 DEBUG [master/fad905551524:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fad905551524:0:becomeActiveMaster-HFileCleaner.large.0-1733834009824,5,FailOnTimeoutGroup] 2024-12-10T12:33:29,824 DEBUG [master/fad905551524:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fad905551524:0:becomeActiveMaster-HFileCleaner.small.0-1733834009824,5,FailOnTimeoutGroup] 2024-12-10T12:33:29,824 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,825 INFO [master/fad905551524:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T12:33:29,826 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,826 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741831_1007 (size=1039) 2024-12-10T12:33:29,843 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37305, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T12:33:29,849 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45051 {}] master.ServerManager(332): Checking decommissioned status of RegionServer fad905551524,38809,1733834007617 2024-12-10T12:33:29,851 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45051 {}] master.ServerManager(486): Registering regionserver=fad905551524,38809,1733834007617 2024-12-10T12:33:29,864 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 2024-12-10T12:33:29,864 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:39921 2024-12-10T12:33:29,864 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-10T12:33:29,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:33:29,877 DEBUG [RS:0;fad905551524:38809 {}] zookeeper.ZKUtil(111): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fad905551524,38809,1733834007617 2024-12-10T12:33:29,878 WARN [RS:0;fad905551524:38809 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T12:33:29,878 INFO [RS:0;fad905551524:38809 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:33:29,878 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/WALs/fad905551524,38809,1733834007617 2024-12-10T12:33:29,880 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fad905551524,38809,1733834007617] 2024-12-10T12:33:29,894 DEBUG [RS:0;fad905551524:38809 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-10T12:33:29,906 INFO [RS:0;fad905551524:38809 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T12:33:29,917 INFO [RS:0;fad905551524:38809 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T12:33:29,920 INFO [RS:0;fad905551524:38809 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T12:33:29,920 INFO [RS:0;fad905551524:38809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,921 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-10T12:33:29,927 INFO [RS:0;fad905551524:38809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,928 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,928 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,928 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,928 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,928 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,929 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fad905551524:0, corePoolSize=2, maxPoolSize=2 2024-12-10T12:33:29,929 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,929 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,929 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,929 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,929 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fad905551524:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:33:29,929 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fad905551524:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:33:29,930 DEBUG [RS:0;fad905551524:38809 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fad905551524:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:33:29,930 INFO [RS:0;fad905551524:38809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,931 INFO [RS:0;fad905551524:38809 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,931 INFO [RS:0;fad905551524:38809 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,931 INFO [RS:0;fad905551524:38809 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,931 INFO [RS:0;fad905551524:38809 {}] hbase.ChoreService(168): Chore ScheduledChore name=fad905551524,38809,1733834007617-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T12:33:29,948 INFO [RS:0;fad905551524:38809 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T12:33:29,950 INFO [RS:0;fad905551524:38809 {}] hbase.ChoreService(168): Chore ScheduledChore name=fad905551524,38809,1733834007617-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:29,968 INFO [RS:0;fad905551524:38809 {}] regionserver.Replication(204): fad905551524,38809,1733834007617 started 2024-12-10T12:33:29,968 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1767): Serving as fad905551524,38809,1733834007617, RpcServer on fad905551524/172.17.0.2:38809, sessionid=0x1000fee7b100001 2024-12-10T12:33:29,969 DEBUG [RS:0;fad905551524:38809 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T12:33:29,969 DEBUG [RS:0;fad905551524:38809 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fad905551524,38809,1733834007617 2024-12-10T12:33:29,969 DEBUG [RS:0;fad905551524:38809 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fad905551524,38809,1733834007617' 2024-12-10T12:33:29,969 DEBUG [RS:0;fad905551524:38809 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T12:33:29,970 DEBUG [RS:0;fad905551524:38809 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T12:33:29,971 DEBUG [RS:0;fad905551524:38809 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T12:33:29,971 DEBUG [RS:0;fad905551524:38809 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T12:33:29,971 DEBUG [RS:0;fad905551524:38809 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fad905551524,38809,1733834007617 2024-12-10T12:33:29,971 DEBUG [RS:0;fad905551524:38809 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fad905551524,38809,1733834007617' 2024-12-10T12:33:29,971 DEBUG [RS:0;fad905551524:38809 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T12:33:29,972 DEBUG [RS:0;fad905551524:38809 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T12:33:29,973 DEBUG [RS:0;fad905551524:38809 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T12:33:29,973 INFO [RS:0;fad905551524:38809 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T12:33:29,973 INFO [RS:0;fad905551524:38809 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T12:33:30,084 INFO [RS:0;fad905551524:38809 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T12:33:30,087 INFO [RS:0;fad905551524:38809 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fad905551524%2C38809%2C1733834007617, suffix=, logDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/WALs/fad905551524,38809,1733834007617, archiveDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/oldWALs, maxLogs=32 2024-12-10T12:33:30,099 DEBUG [RS:0;fad905551524:38809 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/WALs/fad905551524,38809,1733834007617/fad905551524%2C38809%2C1733834007617.1733834010088, exclude list is [], retry=0 2024-12-10T12:33:30,104 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33875,DS-0b8b429d-7aad-4484-b346-eb8afedaf46b,DISK] 2024-12-10T12:33:30,107 INFO [RS:0;fad905551524:38809 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/WALs/fad905551524,38809,1733834007617/fad905551524%2C38809%2C1733834007617.1733834010088 2024-12-10T12:33:30,107 DEBUG [RS:0;fad905551524:38809 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46143:46143)] 2024-12-10T12:33:30,233 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-10T12:33:30,233 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 2024-12-10T12:33:30,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741833_1009 (size=32) 2024-12-10T12:33:30,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:30,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T12:33:30,656 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T12:33:30,656 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:30,657 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:33:30,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T12:33:30,661 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T12:33:30,661 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:30,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:33:30,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T12:33:30,665 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T12:33:30,665 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:30,666 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:33:30,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740 2024-12-10T12:33:30,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740 2024-12-10T12:33:30,671 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:33:30,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-10T12:33:30,676 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:33:30,677 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74153678, jitterRate=0.10497590899467468}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:33:30,679 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-10T12:33:30,679 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-10T12:33:30,679 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-10T12:33:30,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-10T12:33:30,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T12:33:30,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T12:33:30,681 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-10T12:33:30,681 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-10T12:33:30,685 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-10T12:33:30,685 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-10T12:33:30,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T12:33:30,697 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T12:33:30,699 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T12:33:30,853 DEBUG [fad905551524:45051 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T12:33:30,860 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:33:30,866 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fad905551524,38809,1733834007617, state=OPENING 2024-12-10T12:33:30,910 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T12:33:30,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:30,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:30,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:33:30,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:33:30,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=fad905551524,38809,1733834007617}] 2024-12-10T12:33:31,103 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:31,105 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T12:33:31,107 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45726, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T12:33:31,117 INFO [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-10T12:33:31,117 INFO [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:33:31,118 INFO [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T12:33:31,121 INFO [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fad905551524%2C38809%2C1733834007617.meta, suffix=.meta, logDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/WALs/fad905551524,38809,1733834007617, archiveDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/oldWALs, maxLogs=32 2024-12-10T12:33:31,134 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/WALs/fad905551524,38809,1733834007617/fad905551524%2C38809%2C1733834007617.meta.1733834011122.meta, exclude list is [], retry=0 2024-12-10T12:33:31,137 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33875,DS-0b8b429d-7aad-4484-b346-eb8afedaf46b,DISK] 2024-12-10T12:33:31,140 INFO [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/WALs/fad905551524,38809,1733834007617/fad905551524%2C38809%2C1733834007617.meta.1733834011122.meta 2024-12-10T12:33:31,140 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46143:46143)] 2024-12-10T12:33:31,141 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:33:31,142 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T12:33:31,187 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T12:33:31,191 INFO [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T12:33:31,194 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T12:33:31,195 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:31,195 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-10T12:33:31,195 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-10T12:33:31,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T12:33:31,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T12:33:31,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:31,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:33:31,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T12:33:31,202 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T12:33:31,203 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:31,204 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:33:31,204 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T12:33:31,205 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T12:33:31,205 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:31,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:33:31,207 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740 2024-12-10T12:33:31,210 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740 2024-12-10T12:33:31,213 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:33:31,215 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-10T12:33:31,216 INFO [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74747414, jitterRate=0.11382326483726501}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:33:31,218 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-10T12:33:31,223 INFO [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733834011098 2024-12-10T12:33:31,232 DEBUG [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T12:33:31,232 INFO [RS_OPEN_META-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-10T12:33:31,233 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:33:31,235 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fad905551524,38809,1733834007617, state=OPEN 2024-12-10T12:33:31,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:33:31,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:33:31,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:33:31,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:33:31,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T12:33:31,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=fad905551524,38809,1733834007617 in 405 msec 2024-12-10T12:33:31,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T12:33:31,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 648 msec 2024-12-10T12:33:31,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.6010 sec 2024-12-10T12:33:31,351 INFO [master/fad905551524:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733834011351, completionTime=-1 2024-12-10T12:33:31,351 INFO [master/fad905551524:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-10T12:33:31,351 DEBUG [master/fad905551524:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-10T12:33:31,383 DEBUG [hconnection-0x31d7b3b1-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:31,385 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45732, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:31,394 INFO [master/fad905551524:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-10T12:33:31,394 INFO [master/fad905551524:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733834071394 2024-12-10T12:33:31,394 INFO [master/fad905551524:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733834131394 2024-12-10T12:33:31,394 INFO [master/fad905551524:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 42 msec 2024-12-10T12:33:31,431 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fad905551524,45051,1733834006886-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:31,432 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fad905551524,45051,1733834006886-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:31,432 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fad905551524,45051,1733834006886-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:31,433 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fad905551524:45051, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:31,434 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:31,439 DEBUG [master/fad905551524:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-10T12:33:31,443 INFO [master/fad905551524:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-10T12:33:31,444 INFO [master/fad905551524:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T12:33:31,449 DEBUG [master/fad905551524:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-10T12:33:31,452 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T12:33:31,453 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:31,454 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T12:33:31,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741835_1011 (size=358) 2024-12-10T12:33:31,875 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => bab6b7e921719ae441e5ef7d31c45c53, NAME => 'hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 2024-12-10T12:33:31,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741836_1012 (size=42) 2024-12-10T12:33:32,288 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:32,288 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing bab6b7e921719ae441e5ef7d31c45c53, disabling compactions & flushes 2024-12-10T12:33:32,288 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:33:32,288 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:33:32,289 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. after waiting 0 ms 2024-12-10T12:33:32,289 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:33:32,289 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:33:32,289 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for bab6b7e921719ae441e5ef7d31c45c53: 2024-12-10T12:33:32,292 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T12:33:32,299 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733834012294"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733834012294"}]},"ts":"1733834012294"} 2024-12-10T12:33:32,318 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-10T12:33:32,320 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T12:33:32,323 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834012320"}]},"ts":"1733834012320"} 2024-12-10T12:33:32,326 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-10T12:33:32,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=bab6b7e921719ae441e5ef7d31c45c53, ASSIGN}] 2024-12-10T12:33:32,391 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=bab6b7e921719ae441e5ef7d31c45c53, ASSIGN 2024-12-10T12:33:32,394 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=bab6b7e921719ae441e5ef7d31c45c53, ASSIGN; state=OFFLINE, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=false 2024-12-10T12:33:32,545 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=bab6b7e921719ae441e5ef7d31c45c53, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:33:32,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure bab6b7e921719ae441e5ef7d31c45c53, server=fad905551524,38809,1733834007617}] 2024-12-10T12:33:32,709 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:32,721 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:33:32,721 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => bab6b7e921719ae441e5ef7d31c45c53, NAME => 'hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:33:32,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:33:32,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:32,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:33:32,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:33:32,724 INFO [StoreOpener-bab6b7e921719ae441e5ef7d31c45c53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:33:32,726 INFO [StoreOpener-bab6b7e921719ae441e5ef7d31c45c53-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bab6b7e921719ae441e5ef7d31c45c53 columnFamilyName info 2024-12-10T12:33:32,727 DEBUG [StoreOpener-bab6b7e921719ae441e5ef7d31c45c53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:32,727 INFO [StoreOpener-bab6b7e921719ae441e5ef7d31c45c53-1 {}] regionserver.HStore(327): Store=bab6b7e921719ae441e5ef7d31c45c53/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:32,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/namespace/bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:33:32,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/namespace/bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:33:32,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:33:32,737 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/namespace/bab6b7e921719ae441e5ef7d31c45c53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:33:32,738 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened bab6b7e921719ae441e5ef7d31c45c53; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75003414, jitterRate=0.11763796210289001}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T12:33:32,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for bab6b7e921719ae441e5ef7d31c45c53: 2024-12-10T12:33:32,742 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53., pid=6, masterSystemTime=1733834012709 2024-12-10T12:33:32,745 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:33:32,745 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:33:32,746 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=bab6b7e921719ae441e5ef7d31c45c53, regionState=OPEN, openSeqNum=2, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:33:32,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T12:33:32,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure bab6b7e921719ae441e5ef7d31c45c53, server=fad905551524,38809,1733834007617 in 198 msec 2024-12-10T12:33:32,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T12:33:32,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=bab6b7e921719ae441e5ef7d31c45c53, ASSIGN in 366 msec 2024-12-10T12:33:32,759 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T12:33:32,760 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834012759"}]},"ts":"1733834012759"} 2024-12-10T12:33:32,763 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-10T12:33:32,804 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T12:33:32,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3590 sec 2024-12-10T12:33:32,855 DEBUG [master/fad905551524:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-10T12:33:32,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-10T12:33:32,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:32,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:33:32,893 DEBUG [master/fad905551524:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-10T12:33:32,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-10T12:33:32,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 35 msec 2024-12-10T12:33:32,938 DEBUG [master/fad905551524:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-10T12:33:32,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-10T12:33:32,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 33 msec 2024-12-10T12:33:33,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-10T12:33:33,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-10T12:33:33,019 INFO [master/fad905551524:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 5.308sec 2024-12-10T12:33:33,022 INFO [master/fad905551524:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T12:33:33,024 INFO [master/fad905551524:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T12:33:33,027 INFO [master/fad905551524:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T12:33:33,027 INFO [master/fad905551524:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T12:33:33,027 INFO [master/fad905551524:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T12:33:33,028 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fad905551524,45051,1733834006886-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T12:33:33,028 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fad905551524,45051,1733834006886-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T12:33:33,034 DEBUG [master/fad905551524:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-10T12:33:33,034 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T12:33:33,035 INFO [master/fad905551524:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fad905551524,45051,1733834006886-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:33:33,077 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38630296 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@76523d14 2024-12-10T12:33:33,078 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-10T12:33:33,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75444e35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:33,093 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T12:33:33,093 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T12:33:33,102 DEBUG [hconnection-0x25802b45-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:33,111 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45748, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:33,119 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=fad905551524,45051,1733834006886 2024-12-10T12:33:33,133 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMixedAtomicity Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=10166 2024-12-10T12:33:33,142 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T12:33:33,145 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54402, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T12:33:33,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T12:33:33,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:33:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T12:33:33,175 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T12:33:33,175 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:33,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 9 2024-12-10T12:33:33,178 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T12:33:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-10T12:33:33,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741837_1013 (size=963) 2024-12-10T12:33:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-10T12:33:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-10T12:33:33,602 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 2024-12-10T12:33:33,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741838_1014 (size=53) 2024-12-10T12:33:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-10T12:33:34,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:34,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing 646ff6269f37d6ebf1ce7cbee04a0c04, disabling compactions & flushes 2024-12-10T12:33:34,015 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:34,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:34,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. after waiting 0 ms 2024-12-10T12:33:34,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:34,015 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:34,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:34,018 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T12:33:34,019 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733834014018"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733834014018"}]},"ts":"1733834014018"} 2024-12-10T12:33:34,023 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-10T12:33:34,025 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T12:33:34,026 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834014025"}]},"ts":"1733834014025"} 2024-12-10T12:33:34,028 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T12:33:34,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=646ff6269f37d6ebf1ce7cbee04a0c04, ASSIGN}] 2024-12-10T12:33:34,079 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=646ff6269f37d6ebf1ce7cbee04a0c04, ASSIGN 2024-12-10T12:33:34,081 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=646ff6269f37d6ebf1ce7cbee04a0c04, ASSIGN; state=OFFLINE, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=false 2024-12-10T12:33:34,232 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=646ff6269f37d6ebf1ce7cbee04a0c04, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:33:34,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617}] 2024-12-10T12:33:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-10T12:33:34,392 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:34,404 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:34,404 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:33:34,405 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:34,405 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:34,405 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:34,405 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:34,408 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:34,410 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:33:34,411 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 646ff6269f37d6ebf1ce7cbee04a0c04 columnFamilyName A 2024-12-10T12:33:34,411 DEBUG [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:34,412 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] regionserver.HStore(327): Store=646ff6269f37d6ebf1ce7cbee04a0c04/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:34,412 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:34,414 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:33:34,414 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 646ff6269f37d6ebf1ce7cbee04a0c04 columnFamilyName B 2024-12-10T12:33:34,414 DEBUG [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:34,415 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] regionserver.HStore(327): Store=646ff6269f37d6ebf1ce7cbee04a0c04/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:34,415 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:34,417 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:33:34,418 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 646ff6269f37d6ebf1ce7cbee04a0c04 columnFamilyName C 2024-12-10T12:33:34,418 DEBUG [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:34,419 INFO [StoreOpener-646ff6269f37d6ebf1ce7cbee04a0c04-1 {}] regionserver.HStore(327): Store=646ff6269f37d6ebf1ce7cbee04a0c04/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:34,419 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:34,421 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:34,422 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:34,424 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:33:34,427 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:34,430 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:33:34,431 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 646ff6269f37d6ebf1ce7cbee04a0c04; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71684231, jitterRate=0.06817828118801117}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:33:34,432 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:34,434 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., pid=11, masterSystemTime=1733834014392 2024-12-10T12:33:34,436 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:34,437 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:34,437 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=646ff6269f37d6ebf1ce7cbee04a0c04, regionState=OPEN, openSeqNum=2, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:33:34,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-10T12:33:34,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 in 203 msec 2024-12-10T12:33:34,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-10T12:33:34,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=646ff6269f37d6ebf1ce7cbee04a0c04, ASSIGN in 367 msec 2024-12-10T12:33:34,447 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T12:33:34,447 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834014447"}]},"ts":"1733834014447"} 2024-12-10T12:33:34,449 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T12:33:34,461 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T12:33:34,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2910 sec 2024-12-10T12:33:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-10T12:33:35,303 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 9 completed 2024-12-10T12:33:35,310 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e67f019 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6fcb5f29 2024-12-10T12:33:35,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fdf5682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,360 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45764, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T12:33:35,369 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54412, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T12:33:35,375 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5095ba91 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f2091cc 2024-12-10T12:33:35,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d38d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,386 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x12885408 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9bd0964 2024-12-10T12:33:35,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b32f98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,399 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62c43377 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18cb251d 2024-12-10T12:33:35,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736f1673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,413 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x04977266 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@45b55c24 2024-12-10T12:33:35,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bbb5d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,426 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a8f4734 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e52b42a 2024-12-10T12:33:35,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f34ff67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,444 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10c964e8 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9ed28bb 2024-12-10T12:33:35,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b5cad1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,457 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x72e97e4b to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@12a1285d 2024-12-10T12:33:35,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3b736e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,472 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x527c6d40 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@353bc462 2024-12-10T12:33:35,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767a8485, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,488 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c8de680 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@47fe2fa7 2024-12-10T12:33:35,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6502d571, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:33:35,505 DEBUG [hconnection-0x7c3cfe41-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,505 DEBUG [hconnection-0x169ff269-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,505 DEBUG [hconnection-0x47be79fd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:35,511 DEBUG [hconnection-0x1127e63d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,511 DEBUG [hconnection-0x4b969ab9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,513 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45766, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,513 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45768, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,513 DEBUG [hconnection-0x522bb58c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,513 DEBUG [hconnection-0x293c9f96-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,513 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45784, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,515 DEBUG [hconnection-0x34907d80-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees 2024-12-10T12:33:35,517 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45812, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,517 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45800, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,517 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45796, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,518 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:35,520 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45828, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-10T12:33:35,521 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45832, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,521 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:35,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:35,523 DEBUG [hconnection-0x3679459b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:33:35,545 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45842, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:33:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:35,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:33:35,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:35,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:35,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-10T12:33:35,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/93a89651f3014d8eacf0d57e9421c8d9 is 50, key is test_row_0/A:col10/1733834015574/Put/seqid=0 2024-12-10T12:33:35,684 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:35,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-12-10T12:33:35,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:35,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:35,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:35,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:35,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:35,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834075688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:35,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741839_1015 (size=12001) 2024-12-10T12:33:35,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834075691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834075694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/93a89651f3014d8eacf0d57e9421c8d9 2024-12-10T12:33:35,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834075696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834075697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/de8e7f5ec18c458d9e2d53216da35e7a is 50, key is test_row_0/B:col10/1733834015574/Put/seqid=0 2024-12-10T12:33:35,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741840_1016 (size=12001) 2024-12-10T12:33:35,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/de8e7f5ec18c458d9e2d53216da35e7a 2024-12-10T12:33:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-10T12:33:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834075831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834075832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834075831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834075834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834075835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:35,856 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:35,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-12-10T12:33:35,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:35,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:35,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:35,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:35,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/01578f0f6c244cd3a182950b4d89d810 is 50, key is test_row_0/C:col10/1733834015574/Put/seqid=0 2024-12-10T12:33:35,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741841_1017 (size=12001) 2024-12-10T12:33:35,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/01578f0f6c244cd3a182950b4d89d810 2024-12-10T12:33:35,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:35,899 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T12:33:35,901 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T12:33:35,902 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-10T12:33:35,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/93a89651f3014d8eacf0d57e9421c8d9 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/93a89651f3014d8eacf0d57e9421c8d9 2024-12-10T12:33:35,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/93a89651f3014d8eacf0d57e9421c8d9, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T12:33:35,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/de8e7f5ec18c458d9e2d53216da35e7a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/de8e7f5ec18c458d9e2d53216da35e7a 2024-12-10T12:33:35,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/de8e7f5ec18c458d9e2d53216da35e7a, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T12:33:35,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/01578f0f6c244cd3a182950b4d89d810 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/01578f0f6c244cd3a182950b4d89d810 2024-12-10T12:33:35,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/01578f0f6c244cd3a182950b4d89d810, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T12:33:35,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 395ms, sequenceid=13, compaction requested=false 2024-12-10T12:33:35,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:36,036 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:36,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-12-10T12:33:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:36,049 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-10T12:33:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:36,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:36,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:36,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:36,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:36,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:36,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:36,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:36,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834076072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834076075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834076078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834076078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834076081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/703b85bc7fa94857a0529a840f00460a is 50, key is test_row_0/A:col10/1733834015691/Put/seqid=0 2024-12-10T12:33:36,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741842_1018 (size=12001) 2024-12-10T12:33:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-10T12:33:36,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834076180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834076185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834076187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834076187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834076188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834076393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834076395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834076399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834076400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834076402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,536 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/703b85bc7fa94857a0529a840f00460a 2024-12-10T12:33:36,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/2bb800e66edb4a0cb717d8e00f0dccc5 is 50, key is test_row_0/B:col10/1733834015691/Put/seqid=0 2024-12-10T12:33:36,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741843_1019 (size=12001) 2024-12-10T12:33:36,584 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/2bb800e66edb4a0cb717d8e00f0dccc5 2024-12-10T12:33:36,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/4820e4b5874b49e3a54ff74dfcacf325 is 50, key is test_row_0/C:col10/1733834015691/Put/seqid=0 2024-12-10T12:33:36,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741844_1020 (size=12001) 2024-12-10T12:33:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-10T12:33:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834076699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,706 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T12:33:36,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834076705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834076705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834076705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:36,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834076708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,033 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/4820e4b5874b49e3a54ff74dfcacf325 2024-12-10T12:33:37,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/703b85bc7fa94857a0529a840f00460a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/703b85bc7fa94857a0529a840f00460a 2024-12-10T12:33:37,062 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/703b85bc7fa94857a0529a840f00460a, entries=150, sequenceid=38, filesize=11.7 K 2024-12-10T12:33:37,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/2bb800e66edb4a0cb717d8e00f0dccc5 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2bb800e66edb4a0cb717d8e00f0dccc5 2024-12-10T12:33:37,080 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2bb800e66edb4a0cb717d8e00f0dccc5, entries=150, sequenceid=38, filesize=11.7 K 2024-12-10T12:33:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/4820e4b5874b49e3a54ff74dfcacf325 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4820e4b5874b49e3a54ff74dfcacf325 2024-12-10T12:33:37,101 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4820e4b5874b49e3a54ff74dfcacf325, entries=150, sequenceid=38, filesize=11.7 K 2024-12-10T12:33:37,103 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 1054ms, sequenceid=38, compaction requested=false 2024-12-10T12:33:37,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:37,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:37,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=13 2024-12-10T12:33:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=13 2024-12-10T12:33:37,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-12-10T12:33:37,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5820 sec 2024-12-10T12:33:37,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees in 1.5990 sec 2024-12-10T12:33:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:37,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-10T12:33:37,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:37,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:37,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:37,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:37,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:37,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:37,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/a6be2a304b6944179c0189836c0548a0 is 50, key is test_row_0/A:col10/1733834017208/Put/seqid=0 2024-12-10T12:33:37,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741845_1021 (size=12001) 2024-12-10T12:33:37,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834077253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834077253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834077260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834077261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834077261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T12:33:37,302 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T12:33:37,304 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-10T12:33:37,304 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-12-10T12:33:37,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T12:33:37,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T12:33:37,307 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T12:33:37,307 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T12:33:37,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T12:33:37,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T12:33:37,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834077364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834077365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834077368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834077369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834077370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834077569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834077570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834077574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834077575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834077575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-10T12:33:37,635 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 12 completed 2024-12-10T12:33:37,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees 2024-12-10T12:33:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-10T12:33:37,641 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:37,643 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:37,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:37,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/a6be2a304b6944179c0189836c0548a0 2024-12-10T12:33:37,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/417e9c66d2204b6799e6bac035377088 is 50, key is test_row_0/B:col10/1733834017208/Put/seqid=0 2024-12-10T12:33:37,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741846_1022 (size=12001) 2024-12-10T12:33:37,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/417e9c66d2204b6799e6bac035377088 2024-12-10T12:33:37,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/879e8b35efa74e2ea2553217f0caf07f is 50, key is test_row_0/C:col10/1733834017208/Put/seqid=0 2024-12-10T12:33:37,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741847_1023 (size=12001) 2024-12-10T12:33:37,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/879e8b35efa74e2ea2553217f0caf07f 2024-12-10T12:33:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-10T12:33:37,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/a6be2a304b6944179c0189836c0548a0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a6be2a304b6944179c0189836c0548a0 2024-12-10T12:33:37,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a6be2a304b6944179c0189836c0548a0, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T12:33:37,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/417e9c66d2204b6799e6bac035377088 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/417e9c66d2204b6799e6bac035377088 2024-12-10T12:33:37,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/417e9c66d2204b6799e6bac035377088, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T12:33:37,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/879e8b35efa74e2ea2553217f0caf07f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/879e8b35efa74e2ea2553217f0caf07f 2024-12-10T12:33:37,796 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:37,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-12-10T12:33:37,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:37,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(58): pid=15 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:37,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/879e8b35efa74e2ea2553217f0caf07f, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T12:33:37,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 603ms, sequenceid=52, compaction requested=true 2024-12-10T12:33:37,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:37,817 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:37,817 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:37,822 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:37,824 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:37,825 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:37,825 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/de8e7f5ec18c458d9e2d53216da35e7a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2bb800e66edb4a0cb717d8e00f0dccc5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/417e9c66d2204b6799e6bac035377088] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=35.2 K 2024-12-10T12:33:37,827 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting de8e7f5ec18c458d9e2d53216da35e7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733834015537 2024-12-10T12:33:37,828 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:37,828 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2bb800e66edb4a0cb717d8e00f0dccc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733834015691 2024-12-10T12:33:37,828 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:37,828 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:37,829 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/93a89651f3014d8eacf0d57e9421c8d9, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/703b85bc7fa94857a0529a840f00460a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a6be2a304b6944179c0189836c0548a0] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=35.2 K 2024-12-10T12:33:37,829 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 417e9c66d2204b6799e6bac035377088, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834016070 2024-12-10T12:33:37,831 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 93a89651f3014d8eacf0d57e9421c8d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733834015537 2024-12-10T12:33:37,832 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 703b85bc7fa94857a0529a840f00460a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733834015691 2024-12-10T12:33:37,834 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting a6be2a304b6944179c0189836c0548a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834016070 2024-12-10T12:33:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:37,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-10T12:33:37,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:37,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:37,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:37,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:37,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:37,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:37,891 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#9 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:37,892 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#10 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:37,893 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/db510d7bc7a545a9b215579dfbecdcf5 is 50, key is test_row_0/A:col10/1733834017208/Put/seqid=0 2024-12-10T12:33:37,897 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/4e5711c7133340fea324661abb1588e8 is 50, key is test_row_0/B:col10/1733834017208/Put/seqid=0 2024-12-10T12:33:37,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/73b539741ca14e48b573e00b6c4f1dad is 50, key is test_row_0/A:col10/1733834017255/Put/seqid=0 2024-12-10T12:33:37,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741848_1024 (size=12104) 2024-12-10T12:33:37,934 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/db510d7bc7a545a9b215579dfbecdcf5 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/db510d7bc7a545a9b215579dfbecdcf5 2024-12-10T12:33:37,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741849_1025 (size=12104) 2024-12-10T12:33:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-10T12:33:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834077907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,949 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/4e5711c7133340fea324661abb1588e8 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e5711c7133340fea324661abb1588e8 2024-12-10T12:33:37,952 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834077926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-12-10T12:33:37,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:37,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834077929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(58): pid=15 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:37,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834077931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834077933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:37,958 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into db510d7bc7a545a9b215579dfbecdcf5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:37,958 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:37,959 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=13, startTime=1733834017816; duration=0sec 2024-12-10T12:33:37,959 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:37,959 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:37,960 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:37,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741850_1026 (size=14341) 2024-12-10T12:33:37,966 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:37,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/73b539741ca14e48b573e00b6c4f1dad 2024-12-10T12:33:37,966 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:37,966 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:37,967 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/01578f0f6c244cd3a182950b4d89d810, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4820e4b5874b49e3a54ff74dfcacf325, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/879e8b35efa74e2ea2553217f0caf07f] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=35.2 K 2024-12-10T12:33:37,977 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 01578f0f6c244cd3a182950b4d89d810, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733834015537 2024-12-10T12:33:37,978 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4820e4b5874b49e3a54ff74dfcacf325, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733834015691 2024-12-10T12:33:37,978 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into 4e5711c7133340fea324661abb1588e8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:37,978 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:37,979 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=13, startTime=1733834017817; duration=0sec 2024-12-10T12:33:37,979 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 879e8b35efa74e2ea2553217f0caf07f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834016070 2024-12-10T12:33:37,980 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:37,980 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:37,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/0522d462da0b48a8be6a8dadf8cca598 is 50, key is test_row_0/B:col10/1733834017255/Put/seqid=0 2024-12-10T12:33:38,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741851_1027 (size=12001) 2024-12-10T12:33:38,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/0522d462da0b48a8be6a8dadf8cca598 2024-12-10T12:33:38,027 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#13 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:38,029 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/8b54544aa137446a8375f9bc80e03f5a is 50, key is test_row_0/C:col10/1733834017208/Put/seqid=0 2024-12-10T12:33:38,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741852_1028 (size=12104) 2024-12-10T12:33:38,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834078048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,059 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/8b54544aa137446a8375f9bc80e03f5a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8b54544aa137446a8375f9bc80e03f5a 2024-12-10T12:33:38,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/43bc1bfc3e2a4f3a9e6536200e274f8b is 50, key is test_row_0/C:col10/1733834017255/Put/seqid=0 2024-12-10T12:33:38,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834078055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834078056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834078057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834078058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,074 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into 8b54544aa137446a8375f9bc80e03f5a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:38,074 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:38,074 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=13, startTime=1733834017817; duration=0sec 2024-12-10T12:33:38,074 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:38,074 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:38,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741853_1029 (size=12001) 2024-12-10T12:33:38,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/43bc1bfc3e2a4f3a9e6536200e274f8b 2024-12-10T12:33:38,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/73b539741ca14e48b573e00b6c4f1dad as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/73b539741ca14e48b573e00b6c4f1dad 2024-12-10T12:33:38,111 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:38,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-12-10T12:33:38,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:38,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:38,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:38,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(58): pid=15 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:38,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/73b539741ca14e48b573e00b6c4f1dad, entries=200, sequenceid=75, filesize=14.0 K 2024-12-10T12:33:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:38,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/0522d462da0b48a8be6a8dadf8cca598 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0522d462da0b48a8be6a8dadf8cca598 2024-12-10T12:33:38,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0522d462da0b48a8be6a8dadf8cca598, entries=150, sequenceid=75, filesize=11.7 K 2024-12-10T12:33:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/43bc1bfc3e2a4f3a9e6536200e274f8b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/43bc1bfc3e2a4f3a9e6536200e274f8b 2024-12-10T12:33:38,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/43bc1bfc3e2a4f3a9e6536200e274f8b, entries=150, sequenceid=75, filesize=11.7 K 2024-12-10T12:33:38,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 267ms, sequenceid=75, compaction requested=false 2024-12-10T12:33:38,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-10T12:33:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:38,266 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:38,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-12-10T12:33:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:38,268 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-10T12:33:38,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:38,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:38,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:38,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:38,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:38,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:38,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/500ecf0e0ff1429f8ae823b8db3763ee is 50, key is test_row_0/A:col10/1733834017930/Put/seqid=0 2024-12-10T12:33:38,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741854_1030 (size=12001) 2024-12-10T12:33:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834078312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834078314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834078322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834078324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834078328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834078430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834078430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834078434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834078434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834078434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834078640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834078641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834078641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834078641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834078646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,707 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/500ecf0e0ff1429f8ae823b8db3763ee 2024-12-10T12:33:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/5970875972454e45848a6412e0e56176 is 50, key is test_row_0/B:col10/1733834017930/Put/seqid=0 2024-12-10T12:33:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-10T12:33:38,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741855_1031 (size=12001) 2024-12-10T12:33:38,757 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/5970875972454e45848a6412e0e56176 2024-12-10T12:33:38,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/3e4fdbf06a634be0bad28e48fe089559 is 50, key is test_row_0/C:col10/1733834017930/Put/seqid=0 2024-12-10T12:33:38,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741856_1032 (size=12001) 2024-12-10T12:33:38,804 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/3e4fdbf06a634be0bad28e48fe089559 2024-12-10T12:33:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/500ecf0e0ff1429f8ae823b8db3763ee as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/500ecf0e0ff1429f8ae823b8db3763ee 2024-12-10T12:33:38,834 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/500ecf0e0ff1429f8ae823b8db3763ee, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T12:33:38,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/5970875972454e45848a6412e0e56176 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/5970875972454e45848a6412e0e56176 2024-12-10T12:33:38,854 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/5970875972454e45848a6412e0e56176, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T12:33:38,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/3e4fdbf06a634be0bad28e48fe089559 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3e4fdbf06a634be0bad28e48fe089559 2024-12-10T12:33:38,875 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3e4fdbf06a634be0bad28e48fe089559, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T12:33:38,879 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 611ms, sequenceid=93, compaction requested=true 2024-12-10T12:33:38,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:38,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:38,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-10T12:33:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=15 2024-12-10T12:33:38,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14 2024-12-10T12:33:38,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2390 sec 2024-12-10T12:33:38,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees in 1.2490 sec 2024-12-10T12:33:38,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-10T12:33:38,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:38,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:38,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:38,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:38,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:38,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:38,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/d49f4d9cb9944dbcae344b76d3ef3c01 is 50, key is test_row_0/A:col10/1733834018306/Put/seqid=0 2024-12-10T12:33:38,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834078966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834078968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741857_1033 (size=12001) 2024-12-10T12:33:38,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/d49f4d9cb9944dbcae344b76d3ef3c01 2024-12-10T12:33:38,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834078970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834078972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834078974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:38,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/4e2dacfccd1f436bbcac80d03469fe17 is 50, key is test_row_0/B:col10/1733834018306/Put/seqid=0 2024-12-10T12:33:39,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741858_1034 (size=12001) 2024-12-10T12:33:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834079073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834079073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834079078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834079079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834079082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834079275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834079277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834079283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834079283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834079287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/4e2dacfccd1f436bbcac80d03469fe17 2024-12-10T12:33:39,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/8ec2e2d4ef0a4d408b5180198ddf0b0a is 50, key is test_row_0/C:col10/1733834018306/Put/seqid=0 2024-12-10T12:33:39,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741859_1035 (size=12001) 2024-12-10T12:33:39,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/8ec2e2d4ef0a4d408b5180198ddf0b0a 2024-12-10T12:33:39,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/d49f4d9cb9944dbcae344b76d3ef3c01 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d49f4d9cb9944dbcae344b76d3ef3c01 2024-12-10T12:33:39,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d49f4d9cb9944dbcae344b76d3ef3c01, entries=150, sequenceid=117, filesize=11.7 K 2024-12-10T12:33:39,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/4e2dacfccd1f436bbcac80d03469fe17 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e2dacfccd1f436bbcac80d03469fe17 2024-12-10T12:33:39,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e2dacfccd1f436bbcac80d03469fe17, entries=150, sequenceid=117, filesize=11.7 K 2024-12-10T12:33:39,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/8ec2e2d4ef0a4d408b5180198ddf0b0a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8ec2e2d4ef0a4d408b5180198ddf0b0a 2024-12-10T12:33:39,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8ec2e2d4ef0a4d408b5180198ddf0b0a, entries=150, sequenceid=117, filesize=11.7 K 2024-12-10T12:33:39,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 566ms, sequenceid=117, compaction requested=true 2024-12-10T12:33:39,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:39,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:39,515 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:39,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:39,516 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:39,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:39,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:39,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:39,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:39,519 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:39,519 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:39,519 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:39,519 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/db510d7bc7a545a9b215579dfbecdcf5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/73b539741ca14e48b573e00b6c4f1dad, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/500ecf0e0ff1429f8ae823b8db3763ee, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d49f4d9cb9944dbcae344b76d3ef3c01] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=49.3 K 2024-12-10T12:33:39,520 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:39,520 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:39,520 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:39,520 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e5711c7133340fea324661abb1588e8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0522d462da0b48a8be6a8dadf8cca598, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/5970875972454e45848a6412e0e56176, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e2dacfccd1f436bbcac80d03469fe17] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=47.0 K 2024-12-10T12:33:39,523 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting db510d7bc7a545a9b215579dfbecdcf5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834016070 2024-12-10T12:33:39,525 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 73b539741ca14e48b573e00b6c4f1dad, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733834017252 2024-12-10T12:33:39,525 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 4e5711c7133340fea324661abb1588e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834016070 2024-12-10T12:33:39,526 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 0522d462da0b48a8be6a8dadf8cca598, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733834017252 2024-12-10T12:33:39,526 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 500ecf0e0ff1429f8ae823b8db3763ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733834017930 2024-12-10T12:33:39,527 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting d49f4d9cb9944dbcae344b76d3ef3c01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733834018306 2024-12-10T12:33:39,529 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5970875972454e45848a6412e0e56176, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733834017930 2024-12-10T12:33:39,530 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 4e2dacfccd1f436bbcac80d03469fe17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733834018306 2024-12-10T12:33:39,558 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:39,559 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/a092c717d7934ef5870e54999e203702 is 50, key is test_row_0/B:col10/1733834018306/Put/seqid=0 2024-12-10T12:33:39,569 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:39,569 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/5f2b3b51540e4154806cf3fbdd35c5a5 is 50, key is test_row_0/A:col10/1733834018306/Put/seqid=0 2024-12-10T12:33:39,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741860_1036 (size=12241) 2024-12-10T12:33:39,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741861_1037 (size=12241) 2024-12-10T12:33:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:39,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-10T12:33:39,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:39,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:39,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:39,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:39,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:39,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:39,595 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/5f2b3b51540e4154806cf3fbdd35c5a5 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5f2b3b51540e4154806cf3fbdd35c5a5 2024-12-10T12:33:39,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/b70d910e71704dd0a12d3b0488438eee is 50, key is test_row_0/A:col10/1733834018964/Put/seqid=0 2024-12-10T12:33:39,613 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into 5f2b3b51540e4154806cf3fbdd35c5a5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:39,613 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:39,613 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=12, startTime=1733834019515; duration=0sec 2024-12-10T12:33:39,613 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:39,613 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:39,613 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:39,617 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:39,617 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:39,617 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:39,617 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8b54544aa137446a8375f9bc80e03f5a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/43bc1bfc3e2a4f3a9e6536200e274f8b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3e4fdbf06a634be0bad28e48fe089559, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8ec2e2d4ef0a4d408b5180198ddf0b0a] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=47.0 K 2024-12-10T12:33:39,618 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8b54544aa137446a8375f9bc80e03f5a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834016070 2024-12-10T12:33:39,619 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 43bc1bfc3e2a4f3a9e6536200e274f8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733834017252 2024-12-10T12:33:39,620 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3e4fdbf06a634be0bad28e48fe089559, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733834017930 2024-12-10T12:33:39,621 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8ec2e2d4ef0a4d408b5180198ddf0b0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733834018306 2024-12-10T12:33:39,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741862_1038 (size=12101) 2024-12-10T12:33:39,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/b70d910e71704dd0a12d3b0488438eee 2024-12-10T12:33:39,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834079625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834079627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834079630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834079633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834079635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/227afdb4838f431ea2a579eb8d096f76 is 50, key is test_row_0/B:col10/1733834018964/Put/seqid=0 2024-12-10T12:33:39,656 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:39,657 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/242e19ab373f4b1f992f1b4a7220428a is 50, key is test_row_0/C:col10/1733834018306/Put/seqid=0 2024-12-10T12:33:39,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741863_1039 (size=12101) 2024-12-10T12:33:39,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/227afdb4838f431ea2a579eb8d096f76 2024-12-10T12:33:39,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/ead4cd0427d94ece92a391defd59b7f3 is 50, key is test_row_0/C:col10/1733834018964/Put/seqid=0 2024-12-10T12:33:39,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741864_1040 (size=12241) 2024-12-10T12:33:39,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741865_1041 (size=12101) 2024-12-10T12:33:39,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/ead4cd0427d94ece92a391defd59b7f3 2024-12-10T12:33:39,709 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/242e19ab373f4b1f992f1b4a7220428a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/242e19ab373f4b1f992f1b4a7220428a 2024-12-10T12:33:39,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/b70d910e71704dd0a12d3b0488438eee as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b70d910e71704dd0a12d3b0488438eee 2024-12-10T12:33:39,722 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into 242e19ab373f4b1f992f1b4a7220428a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:39,722 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:39,722 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=12, startTime=1733834019517; duration=0sec 2024-12-10T12:33:39,722 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:39,722 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:39,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b70d910e71704dd0a12d3b0488438eee, entries=150, sequenceid=132, filesize=11.8 K 2024-12-10T12:33:39,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/227afdb4838f431ea2a579eb8d096f76 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/227afdb4838f431ea2a579eb8d096f76 2024-12-10T12:33:39,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/227afdb4838f431ea2a579eb8d096f76, entries=150, sequenceid=132, filesize=11.8 K 2024-12-10T12:33:39,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834079739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/ead4cd0427d94ece92a391defd59b7f3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ead4cd0427d94ece92a391defd59b7f3 2024-12-10T12:33:39,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834079739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834079739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834079741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-10T12:33:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834079741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,748 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 14 completed 2024-12-10T12:33:39,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees 2024-12-10T12:33:39,757 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-10T12:33:39,759 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:39,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:39,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ead4cd0427d94ece92a391defd59b7f3, entries=150, sequenceid=132, filesize=11.8 K 2024-12-10T12:33:39,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 180ms, sequenceid=132, compaction requested=false 2024-12-10T12:33:39,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-10T12:33:39,913 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:39,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-12-10T12:33:39,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:39,915 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-10T12:33:39,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:39,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:39,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:39,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:39,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:39,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/59ca3ebb6c9448b3b1879b387f08eeef is 50, key is test_row_0/A:col10/1733834019623/Put/seqid=0 2024-12-10T12:33:39,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741866_1042 (size=12151) 2024-12-10T12:33:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:39,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:39,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834079960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834079960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834079962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834079962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834079963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:39,987 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/a092c717d7934ef5870e54999e203702 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a092c717d7934ef5870e54999e203702 2024-12-10T12:33:40,000 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into a092c717d7934ef5870e54999e203702(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:40,000 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:40,000 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=12, startTime=1733834019515; duration=0sec 2024-12-10T12:33:40,001 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:40,001 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-10T12:33:40,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834080065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834080066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834080068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834080068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834080070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834080269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834080271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834080273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834080274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834080276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,336 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/59ca3ebb6c9448b3b1879b387f08eeef 2024-12-10T12:33:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/3c42427fc45342f9b40d2282a2a90b92 is 50, key is test_row_0/B:col10/1733834019623/Put/seqid=0 2024-12-10T12:33:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-10T12:33:40,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741867_1043 (size=12151) 2024-12-10T12:33:40,378 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/3c42427fc45342f9b40d2282a2a90b92 2024-12-10T12:33:40,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/f42a6b8a37344724a345e01211ef3f8a is 50, key is test_row_0/C:col10/1733834019623/Put/seqid=0 2024-12-10T12:33:40,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741868_1044 (size=12151) 2024-12-10T12:33:40,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834080575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834080575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834080578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834080579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834080583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:40,816 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/f42a6b8a37344724a345e01211ef3f8a 2024-12-10T12:33:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/59ca3ebb6c9448b3b1879b387f08eeef as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/59ca3ebb6c9448b3b1879b387f08eeef 2024-12-10T12:33:40,841 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/59ca3ebb6c9448b3b1879b387f08eeef, entries=150, sequenceid=156, filesize=11.9 K 2024-12-10T12:33:40,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/3c42427fc45342f9b40d2282a2a90b92 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c42427fc45342f9b40d2282a2a90b92 2024-12-10T12:33:40,859 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c42427fc45342f9b40d2282a2a90b92, entries=150, sequenceid=156, filesize=11.9 K 2024-12-10T12:33:40,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/f42a6b8a37344724a345e01211ef3f8a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f42a6b8a37344724a345e01211ef3f8a 2024-12-10T12:33:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-10T12:33:40,872 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f42a6b8a37344724a345e01211ef3f8a, entries=150, sequenceid=156, filesize=11.9 K 2024-12-10T12:33:40,873 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 959ms, sequenceid=156, compaction requested=true 2024-12-10T12:33:40,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:40,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:40,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=17 2024-12-10T12:33:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=17 2024-12-10T12:33:40,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-12-10T12:33:40,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1160 sec 2024-12-10T12:33:40,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees in 1.1260 sec 2024-12-10T12:33:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:41,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-10T12:33:41,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:41,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:41,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:41,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:41,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:41,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:41,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/d88ef7818e6c49ec8416260858064c59 is 50, key is test_row_0/A:col10/1733834021082/Put/seqid=0 2024-12-10T12:33:41,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834081106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834081108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834081110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834081111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834081112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741869_1045 (size=9757) 2024-12-10T12:33:41,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/d88ef7818e6c49ec8416260858064c59 2024-12-10T12:33:41,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/1a340ae90a234917afffb91c8161b10a is 50, key is test_row_0/B:col10/1733834021082/Put/seqid=0 2024-12-10T12:33:41,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741870_1046 (size=9757) 2024-12-10T12:33:41,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/1a340ae90a234917afffb91c8161b10a 2024-12-10T12:33:41,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/8bc76b39f0954771bafe0d49aaae4644 is 50, key is test_row_0/C:col10/1733834021082/Put/seqid=0 2024-12-10T12:33:41,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741871_1047 (size=9757) 2024-12-10T12:33:41,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834081213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834081214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834081217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834081217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834081217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834081419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834081421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834081421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834081422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834081427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/8bc76b39f0954771bafe0d49aaae4644 2024-12-10T12:33:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/d88ef7818e6c49ec8416260858064c59 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d88ef7818e6c49ec8416260858064c59 2024-12-10T12:33:41,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d88ef7818e6c49ec8416260858064c59, entries=100, sequenceid=173, filesize=9.5 K 2024-12-10T12:33:41,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/1a340ae90a234917afffb91c8161b10a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1a340ae90a234917afffb91c8161b10a 2024-12-10T12:33:41,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1a340ae90a234917afffb91c8161b10a, entries=100, sequenceid=173, filesize=9.5 K 2024-12-10T12:33:41,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/8bc76b39f0954771bafe0d49aaae4644 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8bc76b39f0954771bafe0d49aaae4644 2024-12-10T12:33:41,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8bc76b39f0954771bafe0d49aaae4644, entries=100, sequenceid=173, filesize=9.5 K 2024-12-10T12:33:41,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 566ms, sequenceid=173, compaction requested=true 2024-12-10T12:33:41,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:41,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:41,648 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:41,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:41,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:41,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:41,648 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:41,651 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46250 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:41,651 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:41,651 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:41,651 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5f2b3b51540e4154806cf3fbdd35c5a5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b70d910e71704dd0a12d3b0488438eee, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/59ca3ebb6c9448b3b1879b387f08eeef, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d88ef7818e6c49ec8416260858064c59] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=45.2 K 2024-12-10T12:33:41,652 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46250 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:41,652 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:41,652 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:41,652 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a092c717d7934ef5870e54999e203702, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/227afdb4838f431ea2a579eb8d096f76, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c42427fc45342f9b40d2282a2a90b92, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1a340ae90a234917afffb91c8161b10a] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=45.2 K 2024-12-10T12:33:41,654 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting a092c717d7934ef5870e54999e203702, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733834018306 2024-12-10T12:33:41,654 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5f2b3b51540e4154806cf3fbdd35c5a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733834018306 2024-12-10T12:33:41,655 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting b70d910e71704dd0a12d3b0488438eee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733834018964 2024-12-10T12:33:41,655 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 227afdb4838f431ea2a579eb8d096f76, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733834018964 2024-12-10T12:33:41,656 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 59ca3ebb6c9448b3b1879b387f08eeef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733834019623 2024-12-10T12:33:41,657 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 3c42427fc45342f9b40d2282a2a90b92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733834019623 2024-12-10T12:33:41,657 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting d88ef7818e6c49ec8416260858064c59, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733834021081 2024-12-10T12:33:41,658 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1a340ae90a234917afffb91c8161b10a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733834021081 2024-12-10T12:33:41,685 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:41,686 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/8a8d3e6748bd4310910980a2a3342993 is 50, key is test_row_0/A:col10/1733834021082/Put/seqid=0 2024-12-10T12:33:41,693 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:41,694 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/0e33bde1d3b442ce8933175645f26c58 is 50, key is test_row_0/B:col10/1733834021082/Put/seqid=0 2024-12-10T12:33:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:41,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-10T12:33:41,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:41,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:41,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:41,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:41,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:41,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:41,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741872_1048 (size=12527) 2024-12-10T12:33:41,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/a9e8eaa8d5c24024b6c1713798a8aaf3 is 50, key is test_row_0/A:col10/1733834021106/Put/seqid=0 2024-12-10T12:33:41,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741873_1049 (size=12527) 2024-12-10T12:33:41,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834081743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,752 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/8a8d3e6748bd4310910980a2a3342993 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/8a8d3e6748bd4310910980a2a3342993 2024-12-10T12:33:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834081745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834081750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834081750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834081751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,763 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into 8a8d3e6748bd4310910980a2a3342993(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:41,763 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:41,763 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=12, startTime=1733834021648; duration=0sec 2024-12-10T12:33:41,764 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:41,764 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:41,764 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:41,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741874_1050 (size=14541) 2024-12-10T12:33:41,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/a9e8eaa8d5c24024b6c1713798a8aaf3 2024-12-10T12:33:41,769 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46250 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:41,769 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:41,770 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:41,770 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/242e19ab373f4b1f992f1b4a7220428a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ead4cd0427d94ece92a391defd59b7f3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f42a6b8a37344724a345e01211ef3f8a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8bc76b39f0954771bafe0d49aaae4644] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=45.2 K 2024-12-10T12:33:41,770 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 242e19ab373f4b1f992f1b4a7220428a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733834018306 2024-12-10T12:33:41,771 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting ead4cd0427d94ece92a391defd59b7f3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733834018964 2024-12-10T12:33:41,772 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f42a6b8a37344724a345e01211ef3f8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733834019623 2024-12-10T12:33:41,773 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8bc76b39f0954771bafe0d49aaae4644, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733834021081 2024-12-10T12:33:41,792 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#36 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:41,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/e28f5de8d55a4e638288f6ef9f03278c is 50, key is test_row_0/B:col10/1733834021106/Put/seqid=0 2024-12-10T12:33:41,794 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/80d1b99fb64e4c9cb6e8fa7c58b54750 is 50, key is test_row_0/C:col10/1733834021082/Put/seqid=0 2024-12-10T12:33:41,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741875_1051 (size=12151) 2024-12-10T12:33:41,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741876_1052 (size=12527) 2024-12-10T12:33:41,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834081853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834081854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834081854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834081855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834081856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-10T12:33:41,864 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 16 completed 2024-12-10T12:33:41,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees 2024-12-10T12:33:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-10T12:33:41,867 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:41,868 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:41,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-10T12:33:42,035 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:42,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-12-10T12:33:42,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:42,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:42,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:42,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:42,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834082056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834082056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834082058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834082058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834082059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,154 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/0e33bde1d3b442ce8933175645f26c58 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0e33bde1d3b442ce8933175645f26c58 2024-12-10T12:33:42,162 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into 0e33bde1d3b442ce8933175645f26c58(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:42,162 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:42,162 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=12, startTime=1733834021648; duration=0sec 2024-12-10T12:33:42,163 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:42,163 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-10T12:33:42,191 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:42,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-12-10T12:33:42,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:42,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:42,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:42,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/e28f5de8d55a4e638288f6ef9f03278c 2024-12-10T12:33:42,238 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/80d1b99fb64e4c9cb6e8fa7c58b54750 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/80d1b99fb64e4c9cb6e8fa7c58b54750 2024-12-10T12:33:42,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/27066b2e17ba407581c41940aafcd4fd is 50, key is test_row_0/C:col10/1733834021106/Put/seqid=0 2024-12-10T12:33:42,249 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into 80d1b99fb64e4c9cb6e8fa7c58b54750(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:42,249 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:42,249 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=12, startTime=1733834021648; duration=0sec 2024-12-10T12:33:42,249 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:42,249 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:42,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741877_1053 (size=12151) 2024-12-10T12:33:42,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/27066b2e17ba407581c41940aafcd4fd 2024-12-10T12:33:42,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/a9e8eaa8d5c24024b6c1713798a8aaf3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a9e8eaa8d5c24024b6c1713798a8aaf3 2024-12-10T12:33:42,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a9e8eaa8d5c24024b6c1713798a8aaf3, entries=200, sequenceid=194, filesize=14.2 K 2024-12-10T12:33:42,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/e28f5de8d55a4e638288f6ef9f03278c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e28f5de8d55a4e638288f6ef9f03278c 2024-12-10T12:33:42,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e28f5de8d55a4e638288f6ef9f03278c, entries=150, sequenceid=194, filesize=11.9 K 2024-12-10T12:33:42,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/27066b2e17ba407581c41940aafcd4fd as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/27066b2e17ba407581c41940aafcd4fd 2024-12-10T12:33:42,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/27066b2e17ba407581c41940aafcd4fd, entries=150, sequenceid=194, filesize=11.9 K 2024-12-10T12:33:42,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 576ms, sequenceid=194, compaction requested=false 2024-12-10T12:33:42,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:42,345 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:42,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-12-10T12:33:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:42,346 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-10T12:33:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:42,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:42,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:42,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:42,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/a230489259fe47fa9a51661cec85ada5 is 50, key is test_row_0/A:col10/1733834021747/Put/seqid=0 2024-12-10T12:33:42,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741878_1054 (size=12151) 2024-12-10T12:33:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:42,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:42,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834082381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834082382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834082384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834082386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834082386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-10T12:33:42,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834082487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834082488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834082488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834082491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834082491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834082691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834082693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834082693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834082697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834082697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:42,758 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/a230489259fe47fa9a51661cec85ada5 2024-12-10T12:33:42,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/a99ed42c70cd43438fa54e604c2570bc is 50, key is test_row_0/B:col10/1733834021747/Put/seqid=0 2024-12-10T12:33:42,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741879_1055 (size=12151) 2024-12-10T12:33:42,780 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/a99ed42c70cd43438fa54e604c2570bc 2024-12-10T12:33:42,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/6757e4e228b4416886dfbb448233f75a is 50, key is test_row_0/C:col10/1733834021747/Put/seqid=0 2024-12-10T12:33:42,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741880_1056 (size=12151) 2024-12-10T12:33:42,805 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/6757e4e228b4416886dfbb448233f75a 2024-12-10T12:33:42,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/a230489259fe47fa9a51661cec85ada5 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a230489259fe47fa9a51661cec85ada5 2024-12-10T12:33:42,825 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a230489259fe47fa9a51661cec85ada5, entries=150, sequenceid=213, filesize=11.9 K 2024-12-10T12:33:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/a99ed42c70cd43438fa54e604c2570bc as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a99ed42c70cd43438fa54e604c2570bc 2024-12-10T12:33:42,836 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a99ed42c70cd43438fa54e604c2570bc, entries=150, sequenceid=213, filesize=11.9 K 2024-12-10T12:33:42,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/6757e4e228b4416886dfbb448233f75a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6757e4e228b4416886dfbb448233f75a 2024-12-10T12:33:42,851 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6757e4e228b4416886dfbb448233f75a, entries=150, sequenceid=213, filesize=11.9 K 2024-12-10T12:33:42,852 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 506ms, sequenceid=213, compaction requested=true 2024-12-10T12:33:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-10T12:33:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=19 2024-12-10T12:33:42,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-12-10T12:33:42,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 986 msec 2024-12-10T12:33:42,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees in 995 msec 2024-12-10T12:33:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-10T12:33:42,971 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 18 completed 2024-12-10T12:33:42,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees 2024-12-10T12:33:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-10T12:33:42,975 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:42,975 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:42,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:42,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-10T12:33:42,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:42,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:42,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:42,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:42,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:42,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:43,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/60d12d476a5c4f0fa918bb2611c38265 is 50, key is test_row_0/A:col10/1733834022996/Put/seqid=0 2024-12-10T12:33:43,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834083007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834083008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834083009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834083012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834083012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741881_1057 (size=14541) 2024-12-10T12:33:43,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/60d12d476a5c4f0fa918bb2611c38265 2024-12-10T12:33:43,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/aefdfbae8862413d88d77b0a12d5438a is 50, key is test_row_0/B:col10/1733834022996/Put/seqid=0 2024-12-10T12:33:43,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741882_1058 (size=12151) 2024-12-10T12:33:43,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/aefdfbae8862413d88d77b0a12d5438a 2024-12-10T12:33:43,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/2579e32046b343eb9b28611a62eb971d is 50, key is test_row_0/C:col10/1733834022996/Put/seqid=0 2024-12-10T12:33:43,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741883_1059 (size=12151) 2024-12-10T12:33:43,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/2579e32046b343eb9b28611a62eb971d 2024-12-10T12:33:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-10T12:33:43,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/60d12d476a5c4f0fa918bb2611c38265 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/60d12d476a5c4f0fa918bb2611c38265 2024-12-10T12:33:43,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/60d12d476a5c4f0fa918bb2611c38265, entries=200, sequenceid=235, filesize=14.2 K 2024-12-10T12:33:43,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/aefdfbae8862413d88d77b0a12d5438a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/aefdfbae8862413d88d77b0a12d5438a 2024-12-10T12:33:43,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/aefdfbae8862413d88d77b0a12d5438a, entries=150, sequenceid=235, filesize=11.9 K 2024-12-10T12:33:43,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/2579e32046b343eb9b28611a62eb971d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2579e32046b343eb9b28611a62eb971d 2024-12-10T12:33:43,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2579e32046b343eb9b28611a62eb971d, entries=150, sequenceid=235, filesize=11.9 K 2024-12-10T12:33:43,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 104ms, sequenceid=235, compaction requested=true 2024-12-10T12:33:43,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:43,102 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:43,102 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:43,104 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:43,104 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:43,104 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,104 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0e33bde1d3b442ce8933175645f26c58, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e28f5de8d55a4e638288f6ef9f03278c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a99ed42c70cd43438fa54e604c2570bc, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/aefdfbae8862413d88d77b0a12d5438a] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=47.8 K 2024-12-10T12:33:43,104 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:43,104 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:43,104 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,105 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/8a8d3e6748bd4310910980a2a3342993, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a9e8eaa8d5c24024b6c1713798a8aaf3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a230489259fe47fa9a51661cec85ada5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/60d12d476a5c4f0fa918bb2611c38265] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=52.5 K 2024-12-10T12:33:43,105 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8a8d3e6748bd4310910980a2a3342993, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733834019628 2024-12-10T12:33:43,106 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 0e33bde1d3b442ce8933175645f26c58, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733834019628 2024-12-10T12:33:43,106 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting a9e8eaa8d5c24024b6c1713798a8aaf3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733834021106 2024-12-10T12:33:43,107 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting e28f5de8d55a4e638288f6ef9f03278c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733834021106 2024-12-10T12:33:43,107 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting a99ed42c70cd43438fa54e604c2570bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733834021732 2024-12-10T12:33:43,107 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting a230489259fe47fa9a51661cec85ada5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733834021732 2024-12-10T12:33:43,108 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 60d12d476a5c4f0fa918bb2611c38265, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733834022375 2024-12-10T12:33:43,108 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting aefdfbae8862413d88d77b0a12d5438a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733834022375 2024-12-10T12:33:43,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-10T12:33:43,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:43,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:43,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:43,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:43,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:43,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:43,124 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:43,124 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:43,125 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/ed9a9dd68b764267ac6f5294f9e8bcb3 is 50, key is test_row_0/B:col10/1733834022996/Put/seqid=0 2024-12-10T12:33:43,125 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/5c23341d38724c8b8fd6ef3ae13d33b8 is 50, key is test_row_0/A:col10/1733834022996/Put/seqid=0 2024-12-10T12:33:43,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/de303161e20b449c84a3f5a99fe4fe76 is 50, key is test_row_0/A:col10/1733834023010/Put/seqid=0 2024-12-10T12:33:43,127 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:43,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-12-10T12:33:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834083140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834083142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834083141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834083143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834083145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741885_1061 (size=12663) 2024-12-10T12:33:43,166 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/ed9a9dd68b764267ac6f5294f9e8bcb3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/ed9a9dd68b764267ac6f5294f9e8bcb3 2024-12-10T12:33:43,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741886_1062 (size=16931) 2024-12-10T12:33:43,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/de303161e20b449c84a3f5a99fe4fe76 2024-12-10T12:33:43,176 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into ed9a9dd68b764267ac6f5294f9e8bcb3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:43,176 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:43,176 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=12, startTime=1733834023102; duration=0sec 2024-12-10T12:33:43,177 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:43,177 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:43,177 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:43,179 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:43,179 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:43,179 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,179 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/80d1b99fb64e4c9cb6e8fa7c58b54750, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/27066b2e17ba407581c41940aafcd4fd, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6757e4e228b4416886dfbb448233f75a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2579e32046b343eb9b28611a62eb971d] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=47.8 K 2024-12-10T12:33:43,180 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 80d1b99fb64e4c9cb6e8fa7c58b54750, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733834019628 2024-12-10T12:33:43,180 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 27066b2e17ba407581c41940aafcd4fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733834021106 2024-12-10T12:33:43,182 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 6757e4e228b4416886dfbb448233f75a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733834021732 2024-12-10T12:33:43,183 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2579e32046b343eb9b28611a62eb971d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733834022375 2024-12-10T12:33:43,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741884_1060 (size=12663) 2024-12-10T12:33:43,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/90a37381d7ba464da129fc11ac440935 is 50, key is test_row_0/B:col10/1733834023010/Put/seqid=0 2024-12-10T12:33:43,225 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:43,226 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/f5f4f56b3bc4491cae07e9d4ee5b9044 is 50, key is test_row_0/C:col10/1733834022996/Put/seqid=0 2024-12-10T12:33:43,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741887_1063 (size=12151) 2024-12-10T12:33:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834083249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741888_1064 (size=12663) 2024-12-10T12:33:43,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834083253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834083253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834083253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834083253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-10T12:33:43,281 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:43,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-12-10T12:33:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,434 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:43,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-12-10T12:33:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834083452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834083458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834083460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834083460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834083470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-10T12:33:43,588 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:43,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-12-10T12:33:43,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:43,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:43,609 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/5c23341d38724c8b8fd6ef3ae13d33b8 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5c23341d38724c8b8fd6ef3ae13d33b8 2024-12-10T12:33:43,615 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into 5c23341d38724c8b8fd6ef3ae13d33b8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:43,615 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:43,615 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=12, startTime=1733834023102; duration=0sec 2024-12-10T12:33:43,615 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:43,615 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:43,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/90a37381d7ba464da129fc11ac440935 2024-12-10T12:33:43,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/aa205571e4644d07884717572c1ab1bb is 50, key is test_row_0/C:col10/1733834023010/Put/seqid=0 2024-12-10T12:33:43,667 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/f5f4f56b3bc4491cae07e9d4ee5b9044 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f5f4f56b3bc4491cae07e9d4ee5b9044 2024-12-10T12:33:43,676 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into f5f4f56b3bc4491cae07e9d4ee5b9044(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:43,676 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:43,676 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=12, startTime=1733834023102; duration=0sec 2024-12-10T12:33:43,676 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:43,676 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:43,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741889_1065 (size=12151) 2024-12-10T12:33:43,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/aa205571e4644d07884717572c1ab1bb 2024-12-10T12:33:43,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/de303161e20b449c84a3f5a99fe4fe76 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/de303161e20b449c84a3f5a99fe4fe76 2024-12-10T12:33:43,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/de303161e20b449c84a3f5a99fe4fe76, entries=250, sequenceid=251, filesize=16.5 K 2024-12-10T12:33:43,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/90a37381d7ba464da129fc11ac440935 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/90a37381d7ba464da129fc11ac440935 2024-12-10T12:33:43,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/90a37381d7ba464da129fc11ac440935, entries=150, sequenceid=251, filesize=11.9 K 2024-12-10T12:33:43,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/aa205571e4644d07884717572c1ab1bb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aa205571e4644d07884717572c1ab1bb 2024-12-10T12:33:43,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aa205571e4644d07884717572c1ab1bb, entries=150, sequenceid=251, filesize=11.9 K 2024-12-10T12:33:43,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 601ms, sequenceid=251, compaction requested=false 2024-12-10T12:33:43,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:43,742 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:43,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-12-10T12:33:43,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:43,743 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-10T12:33:43,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:43,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:43,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:43,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:43,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:43,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:43,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/106738bba63d4281bb5b02f413ddc163 is 50, key is test_row_0/A:col10/1733834023143/Put/seqid=0 2024-12-10T12:33:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:43,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:43,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741890_1066 (size=12301) 2024-12-10T12:33:43,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834083771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834083771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834083772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834083772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834083773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834083875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834083875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834083875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:43,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834083876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-10T12:33:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834084078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834084079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834084080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834084080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,170 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/106738bba63d4281bb5b02f413ddc163 2024-12-10T12:33:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/3c6dab3376b548b89528938eb0863357 is 50, key is test_row_0/B:col10/1733834023143/Put/seqid=0 2024-12-10T12:33:44,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741891_1067 (size=12301) 2024-12-10T12:33:44,205 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/3c6dab3376b548b89528938eb0863357 2024-12-10T12:33:44,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/a43df02088f946b29998fe55ad6e4c9d is 50, key is test_row_0/C:col10/1733834023143/Put/seqid=0 2024-12-10T12:33:44,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741892_1068 (size=12301) 2024-12-10T12:33:44,229 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/a43df02088f946b29998fe55ad6e4c9d 2024-12-10T12:33:44,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/106738bba63d4281bb5b02f413ddc163 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/106738bba63d4281bb5b02f413ddc163 2024-12-10T12:33:44,242 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/106738bba63d4281bb5b02f413ddc163, entries=150, sequenceid=274, filesize=12.0 K 2024-12-10T12:33:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/3c6dab3376b548b89528938eb0863357 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c6dab3376b548b89528938eb0863357 2024-12-10T12:33:44,250 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c6dab3376b548b89528938eb0863357, entries=150, sequenceid=274, filesize=12.0 K 2024-12-10T12:33:44,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/a43df02088f946b29998fe55ad6e4c9d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a43df02088f946b29998fe55ad6e4c9d 2024-12-10T12:33:44,257 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a43df02088f946b29998fe55ad6e4c9d, entries=150, sequenceid=274, filesize=12.0 K 2024-12-10T12:33:44,259 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 516ms, sequenceid=274, compaction requested=true 2024-12-10T12:33:44,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:44,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:44,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=21 2024-12-10T12:33:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=21 2024-12-10T12:33:44,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=21, resume processing ppid=20 2024-12-10T12:33:44,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2850 sec 2024-12-10T12:33:44,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees in 1.2920 sec 2024-12-10T12:33:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:44,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-10T12:33:44,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:44,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:44,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:44,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:44,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:44,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:44,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/43f72a9830f24e0280c0d8d03c7ced7a is 50, key is test_row_0/A:col10/1733834023771/Put/seqid=0 2024-12-10T12:33:44,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741893_1069 (size=12301) 2024-12-10T12:33:44,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/43f72a9830f24e0280c0d8d03c7ced7a 2024-12-10T12:33:44,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/c5ade30c2cb147cd80f43c1458ea17df is 50, key is test_row_0/B:col10/1733834023771/Put/seqid=0 2024-12-10T12:33:44,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741894_1070 (size=12301) 2024-12-10T12:33:44,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/c5ade30c2cb147cd80f43c1458ea17df 2024-12-10T12:33:44,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834084352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/9fd6a73067304b6ea685e2f6aa7d37e2 is 50, key is test_row_0/C:col10/1733834023771/Put/seqid=0 2024-12-10T12:33:44,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741895_1071 (size=12301) 2024-12-10T12:33:44,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/9fd6a73067304b6ea685e2f6aa7d37e2 2024-12-10T12:33:44,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/43f72a9830f24e0280c0d8d03c7ced7a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/43f72a9830f24e0280c0d8d03c7ced7a 2024-12-10T12:33:44,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/43f72a9830f24e0280c0d8d03c7ced7a, entries=150, sequenceid=291, filesize=12.0 K 2024-12-10T12:33:44,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/c5ade30c2cb147cd80f43c1458ea17df as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/c5ade30c2cb147cd80f43c1458ea17df 2024-12-10T12:33:44,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834084384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834084384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834084386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834084385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/c5ade30c2cb147cd80f43c1458ea17df, entries=150, sequenceid=291, filesize=12.0 K 2024-12-10T12:33:44,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/9fd6a73067304b6ea685e2f6aa7d37e2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/9fd6a73067304b6ea685e2f6aa7d37e2 2024-12-10T12:33:44,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/9fd6a73067304b6ea685e2f6aa7d37e2, entries=150, sequenceid=291, filesize=12.0 K 2024-12-10T12:33:44,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 116ms, sequenceid=291, compaction requested=true 2024-12-10T12:33:44,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:44,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:44,399 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:44,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:44,399 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:44,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:44,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:44,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:44,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:44,401 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:44,401 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:44,401 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:44,402 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/ed9a9dd68b764267ac6f5294f9e8bcb3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/90a37381d7ba464da129fc11ac440935, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c6dab3376b548b89528938eb0863357, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/c5ade30c2cb147cd80f43c1458ea17df] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=48.3 K 2024-12-10T12:33:44,402 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54196 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:44,402 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:44,402 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:44,402 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5c23341d38724c8b8fd6ef3ae13d33b8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/de303161e20b449c84a3f5a99fe4fe76, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/106738bba63d4281bb5b02f413ddc163, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/43f72a9830f24e0280c0d8d03c7ced7a] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=52.9 K 2024-12-10T12:33:44,403 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ed9a9dd68b764267ac6f5294f9e8bcb3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733834022375 2024-12-10T12:33:44,403 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5c23341d38724c8b8fd6ef3ae13d33b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733834022375 2024-12-10T12:33:44,403 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 90a37381d7ba464da129fc11ac440935, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733834023010 2024-12-10T12:33:44,403 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting de303161e20b449c84a3f5a99fe4fe76, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733834023002 2024-12-10T12:33:44,404 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 3c6dab3376b548b89528938eb0863357, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733834023140 2024-12-10T12:33:44,404 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 106738bba63d4281bb5b02f413ddc163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733834023140 2024-12-10T12:33:44,404 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 43f72a9830f24e0280c0d8d03c7ced7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834023770 2024-12-10T12:33:44,405 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting c5ade30c2cb147cd80f43c1458ea17df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834023770 2024-12-10T12:33:44,418 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:44,420 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/2a9b158c2ca4473aa38234e84f24bf21 is 50, key is test_row_0/A:col10/1733834023771/Put/seqid=0 2024-12-10T12:33:44,436 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#58 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:44,437 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/85e685e89a0c49b2b083cecbd778ea1d is 50, key is test_row_0/B:col10/1733834023771/Put/seqid=0 2024-12-10T12:33:44,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741896_1072 (size=12949) 2024-12-10T12:33:44,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741897_1073 (size=12949) 2024-12-10T12:33:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:44,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-10T12:33:44,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:44,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:44,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:44,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:44,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:44,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:44,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/cc9aad7b672e417fac8251568a2f2a50 is 50, key is test_row_0/A:col10/1733834024458/Put/seqid=0 2024-12-10T12:33:44,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741898_1074 (size=12301) 2024-12-10T12:33:44,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/cc9aad7b672e417fac8251568a2f2a50 2024-12-10T12:33:44,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/28855faf257f455ba0af157da06ce040 is 50, key is test_row_0/B:col10/1733834024458/Put/seqid=0 2024-12-10T12:33:44,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741899_1075 (size=12301) 2024-12-10T12:33:44,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834084506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834084609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834084811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,847 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/2a9b158c2ca4473aa38234e84f24bf21 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2a9b158c2ca4473aa38234e84f24bf21 2024-12-10T12:33:44,852 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into 2a9b158c2ca4473aa38234e84f24bf21(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:44,853 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:44,853 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=12, startTime=1733834024399; duration=0sec 2024-12-10T12:33:44,853 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:44,853 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:44,853 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:44,855 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:44,855 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:44,855 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:44,855 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f5f4f56b3bc4491cae07e9d4ee5b9044, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aa205571e4644d07884717572c1ab1bb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a43df02088f946b29998fe55ad6e4c9d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/9fd6a73067304b6ea685e2f6aa7d37e2] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=48.3 K 2024-12-10T12:33:44,856 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f5f4f56b3bc4491cae07e9d4ee5b9044, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733834022375 2024-12-10T12:33:44,856 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting aa205571e4644d07884717572c1ab1bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733834023010 2024-12-10T12:33:44,857 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting a43df02088f946b29998fe55ad6e4c9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733834023140 2024-12-10T12:33:44,857 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9fd6a73067304b6ea685e2f6aa7d37e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834023770 2024-12-10T12:33:44,861 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/85e685e89a0c49b2b083cecbd778ea1d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/85e685e89a0c49b2b083cecbd778ea1d 2024-12-10T12:33:44,870 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:44,871 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/18ec49454ca04535997c6faad4872349 is 50, key is test_row_0/C:col10/1733834023771/Put/seqid=0 2024-12-10T12:33:44,874 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into 85e685e89a0c49b2b083cecbd778ea1d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:44,875 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:44,875 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=12, startTime=1733834024399; duration=0sec 2024-12-10T12:33:44,875 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:44,875 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:44,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741900_1076 (size=12949) 2024-12-10T12:33:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834084890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834084892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/28855faf257f455ba0af157da06ce040 2024-12-10T12:33:44,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834084895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834084895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:44,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/1b907b4ff2694afe9756fd187e80c516 is 50, key is test_row_0/C:col10/1733834024458/Put/seqid=0 2024-12-10T12:33:44,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741901_1077 (size=12301) 2024-12-10T12:33:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-10T12:33:45,081 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 20 completed 2024-12-10T12:33:45,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees 2024-12-10T12:33:45,085 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:45,086 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=22, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-12-10T12:33:45,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:45,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834085114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-12-10T12:33:45,240 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:45,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-12-10T12:33:45,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:45,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:45,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:45,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:45,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:45,291 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/18ec49454ca04535997c6faad4872349 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/18ec49454ca04535997c6faad4872349 2024-12-10T12:33:45,300 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into 18ec49454ca04535997c6faad4872349(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:45,300 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:45,300 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=12, startTime=1733834024400; duration=0sec 2024-12-10T12:33:45,300 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:45,300 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:45,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/1b907b4ff2694afe9756fd187e80c516 2024-12-10T12:33:45,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/cc9aad7b672e417fac8251568a2f2a50 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/cc9aad7b672e417fac8251568a2f2a50 2024-12-10T12:33:45,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/cc9aad7b672e417fac8251568a2f2a50, entries=150, sequenceid=311, filesize=12.0 K 2024-12-10T12:33:45,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/28855faf257f455ba0af157da06ce040 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/28855faf257f455ba0af157da06ce040 2024-12-10T12:33:45,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/28855faf257f455ba0af157da06ce040, entries=150, sequenceid=311, filesize=12.0 K 2024-12-10T12:33:45,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/1b907b4ff2694afe9756fd187e80c516 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1b907b4ff2694afe9756fd187e80c516 2024-12-10T12:33:45,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1b907b4ff2694afe9756fd187e80c516, entries=150, sequenceid=311, filesize=12.0 K 2024-12-10T12:33:45,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 880ms, sequenceid=311, compaction requested=false 2024-12-10T12:33:45,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-12-10T12:33:45,393 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:45,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-12-10T12:33:45,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:45,394 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-10T12:33:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:45,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/f9f5e44900a74e1b841d53cff354999f is 50, key is test_row_0/A:col10/1733834024500/Put/seqid=0 2024-12-10T12:33:45,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741902_1078 (size=12301) 2024-12-10T12:33:45,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:45,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834085677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-12-10T12:33:45,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834085781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:45,812 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/f9f5e44900a74e1b841d53cff354999f 2024-12-10T12:33:45,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/e4e16ea2bda74d7c8f4b13c66804f630 is 50, key is test_row_0/B:col10/1733834024500/Put/seqid=0 2024-12-10T12:33:45,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741903_1079 (size=12301) 2024-12-10T12:33:45,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834085901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:45,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834085903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:45,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834085904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:45,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834085906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:45,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834085984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-12-10T12:33:46,232 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/e4e16ea2bda74d7c8f4b13c66804f630 2024-12-10T12:33:46,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/48a04c42fea546b2a225c3abde793d2e is 50, key is test_row_0/C:col10/1733834024500/Put/seqid=0 2024-12-10T12:33:46,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741904_1080 (size=12301) 2024-12-10T12:33:46,252 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/48a04c42fea546b2a225c3abde793d2e 2024-12-10T12:33:46,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/f9f5e44900a74e1b841d53cff354999f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/f9f5e44900a74e1b841d53cff354999f 2024-12-10T12:33:46,263 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/f9f5e44900a74e1b841d53cff354999f, entries=150, sequenceid=330, filesize=12.0 K 2024-12-10T12:33:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/e4e16ea2bda74d7c8f4b13c66804f630 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e4e16ea2bda74d7c8f4b13c66804f630 2024-12-10T12:33:46,272 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e4e16ea2bda74d7c8f4b13c66804f630, entries=150, sequenceid=330, filesize=12.0 K 2024-12-10T12:33:46,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/48a04c42fea546b2a225c3abde793d2e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/48a04c42fea546b2a225c3abde793d2e 2024-12-10T12:33:46,282 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/48a04c42fea546b2a225c3abde793d2e, entries=150, sequenceid=330, filesize=12.0 K 2024-12-10T12:33:46,284 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 891ms, sequenceid=330, compaction requested=true 2024-12-10T12:33:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=23 2024-12-10T12:33:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=23 2024-12-10T12:33:46,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-12-10T12:33:46,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2000 sec 2024-12-10T12:33:46,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-10T12:33:46,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:46,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:46,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:46,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:46,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:46,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:46,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees in 1.2060 sec 2024-12-10T12:33:46,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/7096fb8a217842cf9fe3821319d30f2d is 50, key is test_row_0/A:col10/1733834026288/Put/seqid=0 2024-12-10T12:33:46,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741905_1081 (size=12301) 2024-12-10T12:33:46,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/7096fb8a217842cf9fe3821319d30f2d 2024-12-10T12:33:46,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/dc569f4db7bc4c988e839fd242611010 is 50, key is test_row_0/B:col10/1733834026288/Put/seqid=0 2024-12-10T12:33:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834086336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:46,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741906_1082 (size=12301) 2024-12-10T12:33:46,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834086439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:46,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834086640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:46,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/dc569f4db7bc4c988e839fd242611010 2024-12-10T12:33:46,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/fa8db5cef9b54fc7ac0ea7293a30007b is 50, key is test_row_0/C:col10/1733834026288/Put/seqid=0 2024-12-10T12:33:46,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741907_1083 (size=12301) 2024-12-10T12:33:46,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834086942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:47,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/fa8db5cef9b54fc7ac0ea7293a30007b 2024-12-10T12:33:47,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/7096fb8a217842cf9fe3821319d30f2d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/7096fb8a217842cf9fe3821319d30f2d 2024-12-10T12:33:47,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/7096fb8a217842cf9fe3821319d30f2d, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T12:33:47,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/dc569f4db7bc4c988e839fd242611010 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/dc569f4db7bc4c988e839fd242611010 2024-12-10T12:33:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-12-10T12:33:47,192 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 22 completed 2024-12-10T12:33:47,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees 2024-12-10T12:33:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-10T12:33:47,195 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:47,196 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=24, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:47,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:47,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/dc569f4db7bc4c988e839fd242611010, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T12:33:47,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/fa8db5cef9b54fc7ac0ea7293a30007b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/fa8db5cef9b54fc7ac0ea7293a30007b 2024-12-10T12:33:47,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/fa8db5cef9b54fc7ac0ea7293a30007b, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T12:33:47,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 918ms, sequenceid=351, compaction requested=true 2024-12-10T12:33:47,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:47,208 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:47,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:47,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:47,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:47,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:47,208 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:47,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:47,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:47,210 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:47,210 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:47,210 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:47,210 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2a9b158c2ca4473aa38234e84f24bf21, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/cc9aad7b672e417fac8251568a2f2a50, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/f9f5e44900a74e1b841d53cff354999f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/7096fb8a217842cf9fe3821319d30f2d] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=48.7 K 2024-12-10T12:33:47,211 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:47,211 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2a9b158c2ca4473aa38234e84f24bf21, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834023770 2024-12-10T12:33:47,211 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:47,211 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:47,211 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/85e685e89a0c49b2b083cecbd778ea1d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/28855faf257f455ba0af157da06ce040, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e4e16ea2bda74d7c8f4b13c66804f630, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/dc569f4db7bc4c988e839fd242611010] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=48.7 K 2024-12-10T12:33:47,211 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting cc9aad7b672e417fac8251568a2f2a50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733834024335 2024-12-10T12:33:47,211 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f9f5e44900a74e1b841d53cff354999f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733834024490 2024-12-10T12:33:47,212 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 85e685e89a0c49b2b083cecbd778ea1d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834023770 2024-12-10T12:33:47,212 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7096fb8a217842cf9fe3821319d30f2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733834025637 2024-12-10T12:33:47,212 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 28855faf257f455ba0af157da06ce040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733834024335 2024-12-10T12:33:47,212 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting e4e16ea2bda74d7c8f4b13c66804f630, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733834024490 2024-12-10T12:33:47,213 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting dc569f4db7bc4c988e839fd242611010, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733834025637 2024-12-10T12:33:47,222 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:47,223 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/1094b09ddb084655bfa49f64aed3265c is 50, key is test_row_0/A:col10/1733834026288/Put/seqid=0 2024-12-10T12:33:47,224 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:47,225 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/11d8c17db37a401d84f00c2352ffb30f is 50, key is test_row_0/B:col10/1733834026288/Put/seqid=0 2024-12-10T12:33:47,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741908_1084 (size=13085) 2024-12-10T12:33:47,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741909_1085 (size=13085) 2024-12-10T12:33:47,252 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/11d8c17db37a401d84f00c2352ffb30f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/11d8c17db37a401d84f00c2352ffb30f 2024-12-10T12:33:47,260 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into 11d8c17db37a401d84f00c2352ffb30f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:47,260 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:47,261 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=12, startTime=1733834027208; duration=0sec 2024-12-10T12:33:47,261 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:47,261 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:47,261 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:33:47,263 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:33:47,263 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:47,263 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:47,264 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/18ec49454ca04535997c6faad4872349, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1b907b4ff2694afe9756fd187e80c516, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/48a04c42fea546b2a225c3abde793d2e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/fa8db5cef9b54fc7ac0ea7293a30007b] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=48.7 K 2024-12-10T12:33:47,265 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 18ec49454ca04535997c6faad4872349, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834023770 2024-12-10T12:33:47,266 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1b907b4ff2694afe9756fd187e80c516, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733834024335 2024-12-10T12:33:47,266 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 48a04c42fea546b2a225c3abde793d2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733834024490 2024-12-10T12:33:47,267 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting fa8db5cef9b54fc7ac0ea7293a30007b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733834025637 2024-12-10T12:33:47,282 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:47,283 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/aad7458921fa4981a812c0cd0a116814 is 50, key is test_row_0/C:col10/1733834026288/Put/seqid=0 2024-12-10T12:33:47,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741910_1086 (size=13085) 2024-12-10T12:33:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-10T12:33:47,305 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/aad7458921fa4981a812c0cd0a116814 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aad7458921fa4981a812c0cd0a116814 2024-12-10T12:33:47,317 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into aad7458921fa4981a812c0cd0a116814(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:47,317 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:47,317 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=12, startTime=1733834027208; duration=0sec 2024-12-10T12:33:47,317 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:47,317 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:47,348 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:47,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-12-10T12:33:47,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:47,349 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-10T12:33:47,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:47,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:47,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:47,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:47,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:47,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:47,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/840598c7b18b4951874a6aa04a6a7f72 is 50, key is test_row_0/A:col10/1733834026332/Put/seqid=0 2024-12-10T12:33:47,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741911_1087 (size=12301) 2024-12-10T12:33:47,368 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/840598c7b18b4951874a6aa04a6a7f72 2024-12-10T12:33:47,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/cee3d259e9774cf284558c47c27faa18 is 50, key is test_row_0/B:col10/1733834026332/Put/seqid=0 2024-12-10T12:33:47,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741912_1088 (size=12301) 2024-12-10T12:33:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:47,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:47,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834087492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-10T12:33:47,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834087594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:47,646 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/1094b09ddb084655bfa49f64aed3265c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/1094b09ddb084655bfa49f64aed3265c 2024-12-10T12:33:47,653 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into 1094b09ddb084655bfa49f64aed3265c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:47,653 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:47,653 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=12, startTime=1733834027208; duration=0sec 2024-12-10T12:33:47,653 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:47,653 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:47,787 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/cee3d259e9774cf284558c47c27faa18 2024-12-10T12:33:47,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/2f885bab09e845a9b8513a9760c791f3 is 50, key is test_row_0/C:col10/1733834026332/Put/seqid=0 2024-12-10T12:33:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-10T12:33:47,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834087798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:47,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741913_1089 (size=12301) 2024-12-10T12:33:47,802 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/2f885bab09e845a9b8513a9760c791f3 2024-12-10T12:33:47,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/840598c7b18b4951874a6aa04a6a7f72 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/840598c7b18b4951874a6aa04a6a7f72 2024-12-10T12:33:47,813 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/840598c7b18b4951874a6aa04a6a7f72, entries=150, sequenceid=368, filesize=12.0 K 2024-12-10T12:33:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/cee3d259e9774cf284558c47c27faa18 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/cee3d259e9774cf284558c47c27faa18 2024-12-10T12:33:47,819 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/cee3d259e9774cf284558c47c27faa18, entries=150, sequenceid=368, filesize=12.0 K 2024-12-10T12:33:47,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/2f885bab09e845a9b8513a9760c791f3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2f885bab09e845a9b8513a9760c791f3 2024-12-10T12:33:47,827 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2f885bab09e845a9b8513a9760c791f3, entries=150, sequenceid=368, filesize=12.0 K 2024-12-10T12:33:47,828 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 479ms, sequenceid=368, compaction requested=false 2024-12-10T12:33:47,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:47,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:47,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=25 2024-12-10T12:33:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=25 2024-12-10T12:33:47,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-12-10T12:33:47,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-12-10T12:33:47,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees in 640 msec 2024-12-10T12:33:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:47,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-10T12:33:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:47,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:47,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:47,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/d2de35c525154739a14caa7d533739e7 is 50, key is test_row_0/A:col10/1733834027491/Put/seqid=0 2024-12-10T12:33:47,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741914_1090 (size=12301) 2024-12-10T12:33:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834087927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834087928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:47,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834087929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:47,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834087930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834088031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834088031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834088034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834088034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834088102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834088234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834088236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834088236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834088238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-10T12:33:48,297 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 24 completed 2024-12-10T12:33:48,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees 2024-12-10T12:33:48,300 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:48,301 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=26, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:48,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-10T12:33:48,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/d2de35c525154739a14caa7d533739e7 2024-12-10T12:33:48,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/1ad2d9ec236948bf8e6c721ff7b8423d is 50, key is test_row_0/B:col10/1733834027491/Put/seqid=0 2024-12-10T12:33:48,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741915_1091 (size=12301) 2024-12-10T12:33:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-10T12:33:48,452 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:48,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-12-10T12:33:48,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:48,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:48,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:48,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834088536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834088538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834088539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834088541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-10T12:33:48,605 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:48,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-12-10T12:33:48,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:48,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:48,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:48,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834088606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:48,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/1ad2d9ec236948bf8e6c721ff7b8423d 2024-12-10T12:33:48,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/70e4c6d99abd4fa792d0fe79ddde6184 is 50, key is test_row_0/C:col10/1733834027491/Put/seqid=0 2024-12-10T12:33:48,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741916_1092 (size=12301) 2024-12-10T12:33:48,760 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:48,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-12-10T12:33:48,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:48,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:48,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:48,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-10T12:33:48,912 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:48,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-12-10T12:33:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:48,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834089040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834089041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834089043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:49,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834089046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:49,066 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:49,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-12-10T12:33:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:49,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:49,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:49,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/70e4c6d99abd4fa792d0fe79ddde6184 2024-12-10T12:33:49,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/d2de35c525154739a14caa7d533739e7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d2de35c525154739a14caa7d533739e7 2024-12-10T12:33:49,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d2de35c525154739a14caa7d533739e7, entries=150, sequenceid=391, filesize=12.0 K 2024-12-10T12:33:49,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/1ad2d9ec236948bf8e6c721ff7b8423d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1ad2d9ec236948bf8e6c721ff7b8423d 2024-12-10T12:33:49,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1ad2d9ec236948bf8e6c721ff7b8423d, entries=150, sequenceid=391, filesize=12.0 K 2024-12-10T12:33:49,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/70e4c6d99abd4fa792d0fe79ddde6184 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/70e4c6d99abd4fa792d0fe79ddde6184 2024-12-10T12:33:49,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/70e4c6d99abd4fa792d0fe79ddde6184, entries=150, sequenceid=391, filesize=12.0 K 2024-12-10T12:33:49,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 1265ms, sequenceid=391, compaction requested=true 2024-12-10T12:33:49,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:49,179 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:49,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:49,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:49,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:49,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:49,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:49,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T12:33:49,179 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:49,181 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:49,181 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:49,181 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:49,181 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/1094b09ddb084655bfa49f64aed3265c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/840598c7b18b4951874a6aa04a6a7f72, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d2de35c525154739a14caa7d533739e7] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=36.8 K 2024-12-10T12:33:49,183 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:49,183 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1094b09ddb084655bfa49f64aed3265c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733834025637 2024-12-10T12:33:49,183 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:49,183 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:49,183 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/11d8c17db37a401d84f00c2352ffb30f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/cee3d259e9774cf284558c47c27faa18, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1ad2d9ec236948bf8e6c721ff7b8423d] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=36.8 K 2024-12-10T12:33:49,183 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 840598c7b18b4951874a6aa04a6a7f72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733834026318 2024-12-10T12:33:49,184 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 11d8c17db37a401d84f00c2352ffb30f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733834025637 2024-12-10T12:33:49,184 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting d2de35c525154739a14caa7d533739e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733834027474 2024-12-10T12:33:49,184 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting cee3d259e9774cf284558c47c27faa18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733834026318 2024-12-10T12:33:49,184 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1ad2d9ec236948bf8e6c721ff7b8423d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733834027474 2024-12-10T12:33:49,192 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:49,193 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/89f8658e86d94e2ea08eac8989a72e4c is 50, key is test_row_0/A:col10/1733834027491/Put/seqid=0 2024-12-10T12:33:49,193 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:49,194 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/67dbbebf5d5943a4a26d88ced1815534 is 50, key is test_row_0/B:col10/1733834027491/Put/seqid=0 2024-12-10T12:33:49,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741918_1094 (size=13187) 2024-12-10T12:33:49,207 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/67dbbebf5d5943a4a26d88ced1815534 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/67dbbebf5d5943a4a26d88ced1815534 2024-12-10T12:33:49,213 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into 67dbbebf5d5943a4a26d88ced1815534(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:49,213 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:49,213 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=13, startTime=1733834029179; duration=0sec 2024-12-10T12:33:49,214 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:49,214 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:49,214 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:49,217 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:49,217 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:49,217 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:49,217 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aad7458921fa4981a812c0cd0a116814, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2f885bab09e845a9b8513a9760c791f3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/70e4c6d99abd4fa792d0fe79ddde6184] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=36.8 K 2024-12-10T12:33:49,218 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting aad7458921fa4981a812c0cd0a116814, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733834025637 2024-12-10T12:33:49,218 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2f885bab09e845a9b8513a9760c791f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733834026318 2024-12-10T12:33:49,218 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:49,218 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 70e4c6d99abd4fa792d0fe79ddde6184, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733834027474 2024-12-10T12:33:49,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-12-10T12:33:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:49,219 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-10T12:33:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:49,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741917_1093 (size=13187) 2024-12-10T12:33:49,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/0ea80d13985c48f1a3eaa44df5440b84 is 50, key is test_row_1/A:col10/1733834027928/Put/seqid=0 2024-12-10T12:33:49,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741919_1095 (size=9857) 2024-12-10T12:33:49,235 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/0ea80d13985c48f1a3eaa44df5440b84 2024-12-10T12:33:49,238 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:49,238 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/d379d8caddb447c9987e3430f6d38719 is 50, key is test_row_0/C:col10/1733834027491/Put/seqid=0 2024-12-10T12:33:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/79d3535e85204752bc4676e045138c0c is 50, key is test_row_1/B:col10/1733834027928/Put/seqid=0 2024-12-10T12:33:49,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741920_1096 (size=13187) 2024-12-10T12:33:49,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741921_1097 (size=9857) 2024-12-10T12:33:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-10T12:33:49,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:49,629 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/89f8658e86d94e2ea08eac8989a72e4c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/89f8658e86d94e2ea08eac8989a72e4c 2024-12-10T12:33:49,635 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into 89f8658e86d94e2ea08eac8989a72e4c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:49,635 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:49,635 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=13, startTime=1733834029179; duration=0sec 2024-12-10T12:33:49,635 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:49,635 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:49,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834089643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:49,657 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/d379d8caddb447c9987e3430f6d38719 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/d379d8caddb447c9987e3430f6d38719 2024-12-10T12:33:49,661 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/79d3535e85204752bc4676e045138c0c 2024-12-10T12:33:49,664 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into d379d8caddb447c9987e3430f6d38719(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:49,665 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:49,665 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=13, startTime=1733834029179; duration=0sec 2024-12-10T12:33:49,665 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:49,665 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:49,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/5a3fd308fe7a4a64ababa60fb7d20e76 is 50, key is test_row_1/C:col10/1733834027928/Put/seqid=0 2024-12-10T12:33:49,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741922_1098 (size=9857) 2024-12-10T12:33:49,679 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/5a3fd308fe7a4a64ababa60fb7d20e76 2024-12-10T12:33:49,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/0ea80d13985c48f1a3eaa44df5440b84 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/0ea80d13985c48f1a3eaa44df5440b84 2024-12-10T12:33:49,693 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/0ea80d13985c48f1a3eaa44df5440b84, entries=100, sequenceid=407, filesize=9.6 K 2024-12-10T12:33:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/79d3535e85204752bc4676e045138c0c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/79d3535e85204752bc4676e045138c0c 2024-12-10T12:33:49,699 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/79d3535e85204752bc4676e045138c0c, entries=100, sequenceid=407, filesize=9.6 K 2024-12-10T12:33:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/5a3fd308fe7a4a64ababa60fb7d20e76 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5a3fd308fe7a4a64ababa60fb7d20e76 2024-12-10T12:33:49,707 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5a3fd308fe7a4a64ababa60fb7d20e76, entries=100, sequenceid=407, filesize=9.6 K 2024-12-10T12:33:49,708 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 489ms, sequenceid=407, compaction requested=false 2024-12-10T12:33:49,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:49,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:49,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=27 2024-12-10T12:33:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=27 2024-12-10T12:33:49,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=27, resume processing ppid=26 2024-12-10T12:33:49,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, ppid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-12-10T12:33:49,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees in 1.4130 sec 2024-12-10T12:33:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:49,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-10T12:33:49,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:49,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:49,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:49,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:49,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:49,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:49,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/b5163dcf45bd4e2d9b6e236633db5fb1 is 50, key is test_row_0/A:col10/1733834029631/Put/seqid=0 2024-12-10T12:33:49,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741923_1099 (size=14741) 2024-12-10T12:33:49,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834089787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:49,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834089890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834090044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:50,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834090049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:50,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834090049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:50,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834090053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:50,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834090094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:50,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/b5163dcf45bd4e2d9b6e236633db5fb1 2024-12-10T12:33:50,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/be0a92d5a23249c1ab7931a1c9327793 is 50, key is test_row_0/B:col10/1733834029631/Put/seqid=0 2024-12-10T12:33:50,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741924_1100 (size=12301) 2024-12-10T12:33:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834090396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-10T12:33:50,406 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 26 completed 2024-12-10T12:33:50,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:50,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees 2024-12-10T12:33:50,408 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:50,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-12-10T12:33:50,409 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=28, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:50,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-12-10T12:33:50,560 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:50,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-12-10T12:33:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:50,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:50,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:50,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/be0a92d5a23249c1ab7931a1c9327793 2024-12-10T12:33:50,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/bac44ed3099b44e0989e579c5d83a806 is 50, key is test_row_0/C:col10/1733834029631/Put/seqid=0 2024-12-10T12:33:50,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741925_1101 (size=12301) 2024-12-10T12:33:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-12-10T12:33:50,713 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:50,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-12-10T12:33:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:50,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:50,866 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:50,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-12-10T12:33:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:50,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:50,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:50,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834090899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:50,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/bac44ed3099b44e0989e579c5d83a806 2024-12-10T12:33:50,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/b5163dcf45bd4e2d9b6e236633db5fb1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b5163dcf45bd4e2d9b6e236633db5fb1 2024-12-10T12:33:50,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b5163dcf45bd4e2d9b6e236633db5fb1, entries=200, sequenceid=431, filesize=14.4 K 2024-12-10T12:33:50,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/be0a92d5a23249c1ab7931a1c9327793 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/be0a92d5a23249c1ab7931a1c9327793 2024-12-10T12:33:51,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/be0a92d5a23249c1ab7931a1c9327793, entries=150, sequenceid=431, filesize=12.0 K 2024-12-10T12:33:51,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/bac44ed3099b44e0989e579c5d83a806 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/bac44ed3099b44e0989e579c5d83a806 2024-12-10T12:33:51,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/bac44ed3099b44e0989e579c5d83a806, entries=150, sequenceid=431, filesize=12.0 K 2024-12-10T12:33:51,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 1264ms, sequenceid=431, compaction requested=true 2024-12-10T12:33:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:51,010 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:51,010 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-12-10T12:33:51,012 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:51,012 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37785 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:51,012 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:51,012 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:51,012 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/67dbbebf5d5943a4a26d88ced1815534, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/79d3535e85204752bc4676e045138c0c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/be0a92d5a23249c1ab7931a1c9327793] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=34.5 K 2024-12-10T12:33:51,012 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:51,012 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:51,012 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/89f8658e86d94e2ea08eac8989a72e4c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/0ea80d13985c48f1a3eaa44df5440b84, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b5163dcf45bd4e2d9b6e236633db5fb1] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=36.9 K 2024-12-10T12:33:51,013 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 67dbbebf5d5943a4a26d88ced1815534, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733834027474 2024-12-10T12:33:51,013 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 89f8658e86d94e2ea08eac8989a72e4c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733834027474 2024-12-10T12:33:51,013 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 79d3535e85204752bc4676e045138c0c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733834027928 2024-12-10T12:33:51,013 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0ea80d13985c48f1a3eaa44df5440b84, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733834027928 2024-12-10T12:33:51,014 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting be0a92d5a23249c1ab7931a1c9327793, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733834029631 2024-12-10T12:33:51,014 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting b5163dcf45bd4e2d9b6e236633db5fb1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733834029631 2024-12-10T12:33:51,018 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:51,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-12-10T12:33:51,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:51,022 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-10T12:33:51,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:51,025 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:51,025 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#87 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:51,026 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/eae7648dbee849388b1377c4a7b93d61 is 50, key is test_row_0/B:col10/1733834029631/Put/seqid=0 2024-12-10T12:33:51,026 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/26186e47287d4cf08c74875aa9270286 is 50, key is test_row_0/A:col10/1733834029631/Put/seqid=0 2024-12-10T12:33:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/97f8d420439a4aaaa3d9a018550ae43e is 50, key is test_row_0/A:col10/1733834029760/Put/seqid=0 2024-12-10T12:33:51,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741927_1103 (size=13289) 2024-12-10T12:33:51,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741926_1102 (size=13289) 2024-12-10T12:33:51,070 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/26186e47287d4cf08c74875aa9270286 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/26186e47287d4cf08c74875aa9270286 2024-12-10T12:33:51,070 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/eae7648dbee849388b1377c4a7b93d61 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/eae7648dbee849388b1377c4a7b93d61 2024-12-10T12:33:51,079 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into eae7648dbee849388b1377c4a7b93d61(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:51,079 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into 26186e47287d4cf08c74875aa9270286(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:51,079 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:51,079 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:51,079 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=13, startTime=1733834031010; duration=0sec 2024-12-10T12:33:51,079 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=13, startTime=1733834031010; duration=0sec 2024-12-10T12:33:51,079 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:51,079 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:51,079 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:51,079 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:51,079 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:51,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741928_1104 (size=12301) 2024-12-10T12:33:51,081 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:51,081 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:51,082 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:51,082 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/d379d8caddb447c9987e3430f6d38719, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5a3fd308fe7a4a64ababa60fb7d20e76, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/bac44ed3099b44e0989e579c5d83a806] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=34.5 K 2024-12-10T12:33:51,082 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting d379d8caddb447c9987e3430f6d38719, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733834027474 2024-12-10T12:33:51,082 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/97f8d420439a4aaaa3d9a018550ae43e 2024-12-10T12:33:51,083 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5a3fd308fe7a4a64ababa60fb7d20e76, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733834027928 2024-12-10T12:33:51,084 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting bac44ed3099b44e0989e579c5d83a806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733834029631 2024-12-10T12:33:51,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/2aa63874a6a84e3aaff9d49e9fe24263 is 50, key is test_row_0/B:col10/1733834029760/Put/seqid=0 2024-12-10T12:33:51,097 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:51,098 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/28471967f1454463bc27b69bbde4afa0 is 50, key is test_row_0/C:col10/1733834029631/Put/seqid=0 2024-12-10T12:33:51,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741929_1105 (size=12301) 2024-12-10T12:33:51,106 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/2aa63874a6a84e3aaff9d49e9fe24263 2024-12-10T12:33:51,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/3b5a93e369454adf8fea226aef7bf81a is 50, key is test_row_0/C:col10/1733834029760/Put/seqid=0 2024-12-10T12:33:51,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741930_1106 (size=13289) 2024-12-10T12:33:51,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741931_1107 (size=12301) 2024-12-10T12:33:51,129 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/28471967f1454463bc27b69bbde4afa0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/28471967f1454463bc27b69bbde4afa0 2024-12-10T12:33:51,131 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/3b5a93e369454adf8fea226aef7bf81a 2024-12-10T12:33:51,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/97f8d420439a4aaaa3d9a018550ae43e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/97f8d420439a4aaaa3d9a018550ae43e 2024-12-10T12:33:51,143 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into 28471967f1454463bc27b69bbde4afa0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:51,143 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:51,143 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=13, startTime=1733834031011; duration=0sec 2024-12-10T12:33:51,143 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:51,143 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:51,146 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/97f8d420439a4aaaa3d9a018550ae43e, entries=150, sequenceid=445, filesize=12.0 K 2024-12-10T12:33:51,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/2aa63874a6a84e3aaff9d49e9fe24263 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2aa63874a6a84e3aaff9d49e9fe24263 2024-12-10T12:33:51,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2aa63874a6a84e3aaff9d49e9fe24263, entries=150, sequenceid=445, filesize=12.0 K 2024-12-10T12:33:51,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/3b5a93e369454adf8fea226aef7bf81a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3b5a93e369454adf8fea226aef7bf81a 2024-12-10T12:33:51,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3b5a93e369454adf8fea226aef7bf81a, entries=150, sequenceid=445, filesize=12.0 K 2024-12-10T12:33:51,167 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 145ms, sequenceid=445, compaction requested=false 2024-12-10T12:33:51,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:51,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:51,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=29 2024-12-10T12:33:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=29 2024-12-10T12:33:51,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-12-10T12:33:51,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 760 msec 2024-12-10T12:33:51,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees in 767 msec 2024-12-10T12:33:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-12-10T12:33:51,512 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 28 completed 2024-12-10T12:33:51,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees 2024-12-10T12:33:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-12-10T12:33:51,514 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=30, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:51,515 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=30, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:51,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-12-10T12:33:51,667 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:51,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-12-10T12:33:51,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:51,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:51,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:51,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=31 2024-12-10T12:33:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=31 2024-12-10T12:33:51,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-12-10T12:33:51,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-10T12:33:51,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees in 158 msec 2024-12-10T12:33:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-12-10T12:33:51,816 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 30 completed 2024-12-10T12:33:51,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-12-10T12:33:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-12-10T12:33:51,862 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:51,862 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:51,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:51,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:33:51,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:51,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:51,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:51,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:51,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:51,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:51,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/2365b44310c14e268050038db7bbeb5f is 50, key is test_row_0/A:col10/1733834031910/Put/seqid=0 2024-12-10T12:33:51,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741932_1108 (size=12301) 2024-12-10T12:33:51,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/2365b44310c14e268050038db7bbeb5f 2024-12-10T12:33:51,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/19b83d29d1eb44aa8d91da824e3a02fa is 50, key is test_row_0/B:col10/1733834031910/Put/seqid=0 2024-12-10T12:33:51,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741933_1109 (size=12301) 2024-12-10T12:33:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-12-10T12:33:51,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834091966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,014 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:52,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T12:33:52,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:52,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45784 deadline: 1733834092050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,052 DEBUG [Thread-157 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4124 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., hostname=fad905551524,38809,1733834007617, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:33:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45842 deadline: 1733834092050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45796 deadline: 1733834092050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,053 DEBUG [Thread-149 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4124 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., hostname=fad905551524,38809,1733834007617, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:33:52,053 DEBUG [Thread-155 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4125 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., hostname=fad905551524,38809,1733834007617, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:33:52,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45828 deadline: 1733834092070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834092070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,071 DEBUG [Thread-153 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4142 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., hostname=fad905551524,38809,1733834007617, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:33:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-12-10T12:33:52,167 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:52,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T12:33:52,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:52,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:52,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:52,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834092272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,319 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:52,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T12:33:52,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:52,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:52,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:33:52,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/19b83d29d1eb44aa8d91da824e3a02fa 2024-12-10T12:33:52,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/6bbcd6b7d3334effad7c076c07dc1296 is 50, key is test_row_0/C:col10/1733834031910/Put/seqid=0 2024-12-10T12:33:52,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741934_1110 (size=12301) 2024-12-10T12:33:52,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/6bbcd6b7d3334effad7c076c07dc1296 2024-12-10T12:33:52,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/2365b44310c14e268050038db7bbeb5f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2365b44310c14e268050038db7bbeb5f 2024-12-10T12:33:52,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2365b44310c14e268050038db7bbeb5f, entries=150, sequenceid=460, filesize=12.0 K 2024-12-10T12:33:52,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/19b83d29d1eb44aa8d91da824e3a02fa as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/19b83d29d1eb44aa8d91da824e3a02fa 2024-12-10T12:33:52,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/19b83d29d1eb44aa8d91da824e3a02fa, entries=150, sequenceid=460, filesize=12.0 K 2024-12-10T12:33:52,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/6bbcd6b7d3334effad7c076c07dc1296 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6bbcd6b7d3334effad7c076c07dc1296 2024-12-10T12:33:52,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6bbcd6b7d3334effad7c076c07dc1296, entries=150, sequenceid=460, filesize=12.0 K 2024-12-10T12:33:52,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 482ms, sequenceid=460, compaction requested=true 2024-12-10T12:33:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:52,394 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:52,394 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:52,395 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:52,395 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:52,396 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:52,396 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:52,396 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,396 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,396 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/eae7648dbee849388b1377c4a7b93d61, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2aa63874a6a84e3aaff9d49e9fe24263, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/19b83d29d1eb44aa8d91da824e3a02fa] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=37.0 K 2024-12-10T12:33:52,396 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/26186e47287d4cf08c74875aa9270286, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/97f8d420439a4aaaa3d9a018550ae43e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2365b44310c14e268050038db7bbeb5f] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=37.0 K 2024-12-10T12:33:52,396 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting eae7648dbee849388b1377c4a7b93d61, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733834029631 2024-12-10T12:33:52,396 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 26186e47287d4cf08c74875aa9270286, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733834029631 2024-12-10T12:33:52,397 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2aa63874a6a84e3aaff9d49e9fe24263, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733834029760 2024-12-10T12:33:52,397 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 97f8d420439a4aaaa3d9a018550ae43e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733834029760 2024-12-10T12:33:52,397 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 19b83d29d1eb44aa8d91da824e3a02fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733834031906 2024-12-10T12:33:52,398 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2365b44310c14e268050038db7bbeb5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733834031906 2024-12-10T12:33:52,406 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:52,407 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/74dc7d2b52e84bda9c3da902c577be97 is 50, key is test_row_0/B:col10/1733834031910/Put/seqid=0 2024-12-10T12:33:52,413 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:52,413 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/fb5f61360b3b47c38c4afb31ff0f2119 is 50, key is test_row_0/A:col10/1733834031910/Put/seqid=0 2024-12-10T12:33:52,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741935_1111 (size=13391) 2024-12-10T12:33:52,431 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/74dc7d2b52e84bda9c3da902c577be97 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/74dc7d2b52e84bda9c3da902c577be97 2024-12-10T12:33:52,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741936_1112 (size=13391) 2024-12-10T12:33:52,457 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into 74dc7d2b52e84bda9c3da902c577be97(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:52,458 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:52,458 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=13, startTime=1733834032394; duration=0sec 2024-12-10T12:33:52,458 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:52,458 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:52,458 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:52,460 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:52,460 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:52,461 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,461 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/28471967f1454463bc27b69bbde4afa0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3b5a93e369454adf8fea226aef7bf81a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6bbcd6b7d3334effad7c076c07dc1296] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=37.0 K 2024-12-10T12:33:52,461 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 28471967f1454463bc27b69bbde4afa0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733834029631 2024-12-10T12:33:52,462 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 3b5a93e369454adf8fea226aef7bf81a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733834029760 2024-12-10T12:33:52,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 6bbcd6b7d3334effad7c076c07dc1296, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733834031906 2024-12-10T12:33:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-12-10T12:33:52,465 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/fb5f61360b3b47c38c4afb31ff0f2119 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/fb5f61360b3b47c38c4afb31ff0f2119 2024-12-10T12:33:52,472 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:52,473 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into fb5f61360b3b47c38c4afb31ff0f2119(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:52,473 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:52,473 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=13, startTime=1733834032394; duration=0sec 2024-12-10T12:33:52,474 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:52,474 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:52,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-12-10T12:33:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,475 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:33:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:52,479 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:52,480 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/a4bfd8fc7f0a4dcba32ec5660c3772c2 is 50, key is test_row_0/C:col10/1733834031910/Put/seqid=0 2024-12-10T12:33:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/6895cf7e9dd94dd8bda98a5b7bd8b7da is 50, key is test_row_0/A:col10/1733834031965/Put/seqid=0 2024-12-10T12:33:52,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741938_1114 (size=12301) 2024-12-10T12:33:52,498 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/6895cf7e9dd94dd8bda98a5b7bd8b7da 2024-12-10T12:33:52,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741937_1113 (size=13391) 2024-12-10T12:33:52,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/644bf27af9f54cf5b780b6f508e46118 is 50, key is test_row_0/B:col10/1733834031965/Put/seqid=0 2024-12-10T12:33:52,509 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/a4bfd8fc7f0a4dcba32ec5660c3772c2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a4bfd8fc7f0a4dcba32ec5660c3772c2 2024-12-10T12:33:52,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741939_1115 (size=12301) 2024-12-10T12:33:52,519 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into a4bfd8fc7f0a4dcba32ec5660c3772c2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:52,519 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:52,519 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=13, startTime=1733834032394; duration=0sec 2024-12-10T12:33:52,519 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:52,519 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:52,526 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/644bf27af9f54cf5b780b6f508e46118 2024-12-10T12:33:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/ca1bbcf4b39444d2af697bec7d76eab0 is 50, key is test_row_0/C:col10/1733834031965/Put/seqid=0 2024-12-10T12:33:52,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741940_1116 (size=12301) 2024-12-10T12:33:52,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:52,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834092592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834092695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834092898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:52,946 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/ca1bbcf4b39444d2af697bec7d76eab0 2024-12-10T12:33:52,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/6895cf7e9dd94dd8bda98a5b7bd8b7da as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/6895cf7e9dd94dd8bda98a5b7bd8b7da 2024-12-10T12:33:52,961 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/6895cf7e9dd94dd8bda98a5b7bd8b7da, entries=150, sequenceid=487, filesize=12.0 K 2024-12-10T12:33:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/644bf27af9f54cf5b780b6f508e46118 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/644bf27af9f54cf5b780b6f508e46118 2024-12-10T12:33:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-12-10T12:33:52,967 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/644bf27af9f54cf5b780b6f508e46118, entries=150, sequenceid=487, filesize=12.0 K 2024-12-10T12:33:52,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/ca1bbcf4b39444d2af697bec7d76eab0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ca1bbcf4b39444d2af697bec7d76eab0 2024-12-10T12:33:52,974 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ca1bbcf4b39444d2af697bec7d76eab0, entries=150, sequenceid=487, filesize=12.0 K 2024-12-10T12:33:52,975 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 500ms, sequenceid=487, compaction requested=false 2024-12-10T12:33:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-10T12:33:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=33 2024-12-10T12:33:52,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=33, resume processing ppid=32 2024-12-10T12:33:52,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, ppid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1150 sec 2024-12-10T12:33:52,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 1.1620 sec 2024-12-10T12:33:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:53,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-10T12:33:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:53,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/48ecc529d80145b49325e5197f973823 is 50, key is test_row_0/A:col10/1733834032586/Put/seqid=0 2024-12-10T12:33:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741941_1117 (size=14741) 2024-12-10T12:33:53,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 345 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834093239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:53,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834093343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:53,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 349 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834093545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:53,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/48ecc529d80145b49325e5197f973823 2024-12-10T12:33:53,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/b0fee64d35fc42b39dd5667edeab5344 is 50, key is test_row_0/B:col10/1733834032586/Put/seqid=0 2024-12-10T12:33:53,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741942_1118 (size=12301) 2024-12-10T12:33:53,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/b0fee64d35fc42b39dd5667edeab5344 2024-12-10T12:33:53,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/5f5ae41f78924e569cc444f9792e4229 is 50, key is test_row_0/C:col10/1733834032586/Put/seqid=0 2024-12-10T12:33:53,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741943_1119 (size=12301) 2024-12-10T12:33:53,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834093849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-12-10T12:33:53,966 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 32 completed 2024-12-10T12:33:53,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:33:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-12-10T12:33:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-10T12:33:53,968 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:33:53,969 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:33:53,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:33:54,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/5f5ae41f78924e569cc444f9792e4229 2024-12-10T12:33:54,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/48ecc529d80145b49325e5197f973823 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/48ecc529d80145b49325e5197f973823 2024-12-10T12:33:54,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/48ecc529d80145b49325e5197f973823, entries=200, sequenceid=500, filesize=14.4 K 2024-12-10T12:33:54,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/b0fee64d35fc42b39dd5667edeab5344 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/b0fee64d35fc42b39dd5667edeab5344 2024-12-10T12:33:54,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/b0fee64d35fc42b39dd5667edeab5344, entries=150, sequenceid=500, filesize=12.0 K 2024-12-10T12:33:54,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/5f5ae41f78924e569cc444f9792e4229 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5f5ae41f78924e569cc444f9792e4229 2024-12-10T12:33:54,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5f5ae41f78924e569cc444f9792e4229, entries=150, sequenceid=500, filesize=12.0 K 2024-12-10T12:33:54,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 856ms, sequenceid=500, compaction requested=true 2024-12-10T12:33:54,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:54,060 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:33:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:54,060 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:33:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:54,061 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:54,061 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/A is initiating minor compaction (all files) 2024-12-10T12:33:54,061 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:54,061 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/A in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:54,061 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/B is initiating minor compaction (all files) 2024-12-10T12:33:54,061 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/B in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:54,061 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/fb5f61360b3b47c38c4afb31ff0f2119, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/6895cf7e9dd94dd8bda98a5b7bd8b7da, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/48ecc529d80145b49325e5197f973823] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=39.5 K 2024-12-10T12:33:54,061 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/74dc7d2b52e84bda9c3da902c577be97, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/644bf27af9f54cf5b780b6f508e46118, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/b0fee64d35fc42b39dd5667edeab5344] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=37.1 K 2024-12-10T12:33:54,061 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting fb5f61360b3b47c38c4afb31ff0f2119, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733834031906 2024-12-10T12:33:54,061 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 74dc7d2b52e84bda9c3da902c577be97, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733834031906 2024-12-10T12:33:54,063 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 644bf27af9f54cf5b780b6f508e46118, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733834031955 2024-12-10T12:33:54,063 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6895cf7e9dd94dd8bda98a5b7bd8b7da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733834031955 2024-12-10T12:33:54,063 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting b0fee64d35fc42b39dd5667edeab5344, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733834032586 2024-12-10T12:33:54,063 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 48ecc529d80145b49325e5197f973823, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733834032586 2024-12-10T12:33:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-10T12:33:54,071 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#A#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:54,071 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/7705315176964d8c9545ebd8a799c900 is 50, key is test_row_0/A:col10/1733834032586/Put/seqid=0 2024-12-10T12:33:54,073 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:54,073 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/76caccdb5e3d4653a36e990023290473 is 50, key is test_row_0/B:col10/1733834032586/Put/seqid=0 2024-12-10T12:33:54,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741944_1120 (size=13493) 2024-12-10T12:33:54,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741945_1121 (size=13493) 2024-12-10T12:33:54,082 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/76caccdb5e3d4653a36e990023290473 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/76caccdb5e3d4653a36e990023290473 2024-12-10T12:33:54,087 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/B of 646ff6269f37d6ebf1ce7cbee04a0c04 into 76caccdb5e3d4653a36e990023290473(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:54,087 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:54,088 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/B, priority=13, startTime=1733834034060; duration=0sec 2024-12-10T12:33:54,088 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:33:54,088 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:54,088 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:33:54,089 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:33:54,089 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 646ff6269f37d6ebf1ce7cbee04a0c04/C is initiating minor compaction (all files) 2024-12-10T12:33:54,090 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 646ff6269f37d6ebf1ce7cbee04a0c04/C in TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:54,090 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a4bfd8fc7f0a4dcba32ec5660c3772c2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ca1bbcf4b39444d2af697bec7d76eab0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5f5ae41f78924e569cc444f9792e4229] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp, totalSize=37.1 K 2024-12-10T12:33:54,090 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting a4bfd8fc7f0a4dcba32ec5660c3772c2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733834031906 2024-12-10T12:33:54,090 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ca1bbcf4b39444d2af697bec7d76eab0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733834031955 2024-12-10T12:33:54,091 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5f5ae41f78924e569cc444f9792e4229, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733834032586 2024-12-10T12:33:54,101 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 646ff6269f37d6ebf1ce7cbee04a0c04#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:33:54,101 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/7825cd03c0a64961a3d5f0b071716bf6 is 50, key is test_row_0/C:col10/1733834032586/Put/seqid=0 2024-12-10T12:33:54,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741946_1122 (size=13493) 2024-12-10T12:33:54,120 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:54,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-10T12:33:54,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:54,121 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-10T12:33:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:54,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/645a52c0020249288cc97e6dd534f5a7 is 50, key is test_row_0/A:col10/1733834033234/Put/seqid=0 2024-12-10T12:33:54,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741947_1123 (size=12301) 2024-12-10T12:33:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-10T12:33:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:54,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. as already flushing 2024-12-10T12:33:54,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 362 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834094365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:54,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 364 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834094480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:54,486 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/7705315176964d8c9545ebd8a799c900 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/7705315176964d8c9545ebd8a799c900 2024-12-10T12:33:54,491 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/A of 646ff6269f37d6ebf1ce7cbee04a0c04 into 7705315176964d8c9545ebd8a799c900(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:54,491 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:54,491 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/A, priority=13, startTime=1733834034060; duration=0sec 2024-12-10T12:33:54,491 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:54,491 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:54,512 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/7825cd03c0a64961a3d5f0b071716bf6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/7825cd03c0a64961a3d5f0b071716bf6 2024-12-10T12:33:54,518 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 646ff6269f37d6ebf1ce7cbee04a0c04/C of 646ff6269f37d6ebf1ce7cbee04a0c04 into 7825cd03c0a64961a3d5f0b071716bf6(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:33:54,518 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:54,518 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04., storeName=646ff6269f37d6ebf1ce7cbee04a0c04/C, priority=13, startTime=1733834034060; duration=0sec 2024-12-10T12:33:54,518 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:54,518 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:54,529 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/645a52c0020249288cc97e6dd534f5a7 2024-12-10T12:33:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/9d1fbb98a6a5426bac2e54b9b72e7258 is 50, key is test_row_0/B:col10/1733834033234/Put/seqid=0 2024-12-10T12:33:54,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741948_1124 (size=12301) 2024-12-10T12:33:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-10T12:33:54,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 366 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834094682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:54,943 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/9d1fbb98a6a5426bac2e54b9b72e7258 2024-12-10T12:33:54,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/82e58d7869b84ce6be2cada0d624fecb is 50, key is test_row_0/C:col10/1733834033234/Put/seqid=0 2024-12-10T12:33:54,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741949_1125 (size=12301) 2024-12-10T12:33:54,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:33:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 368 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45832 deadline: 1733834094987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 2024-12-10T12:33:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-10T12:33:55,368 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/82e58d7869b84ce6be2cada0d624fecb 2024-12-10T12:33:55,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/645a52c0020249288cc97e6dd534f5a7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/645a52c0020249288cc97e6dd534f5a7 2024-12-10T12:33:55,378 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/645a52c0020249288cc97e6dd534f5a7, entries=150, sequenceid=525, filesize=12.0 K 2024-12-10T12:33:55,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/9d1fbb98a6a5426bac2e54b9b72e7258 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/9d1fbb98a6a5426bac2e54b9b72e7258 2024-12-10T12:33:55,383 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/9d1fbb98a6a5426bac2e54b9b72e7258, entries=150, sequenceid=525, filesize=12.0 K 2024-12-10T12:33:55,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/82e58d7869b84ce6be2cada0d624fecb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/82e58d7869b84ce6be2cada0d624fecb 2024-12-10T12:33:55,388 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/82e58d7869b84ce6be2cada0d624fecb, entries=150, sequenceid=525, filesize=12.0 K 2024-12-10T12:33:55,388 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 1268ms, sequenceid=525, compaction requested=false 2024-12-10T12:33:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-10T12:33:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=35 2024-12-10T12:33:55,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-12-10T12:33:55,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-12-10T12:33:55,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.4240 sec 2024-12-10T12:33:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:55,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:33:55,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:55,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:55,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:55,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:55,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:55,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:55,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/343ff519af914876ba236772edf74800 is 50, key is test_row_0/A:col10/1733834035492/Put/seqid=0 2024-12-10T12:33:55,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741950_1126 (size=14741) 2024-12-10T12:33:55,505 DEBUG [Thread-151 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x12885408 to 127.0.0.1:64063 2024-12-10T12:33:55,505 DEBUG [Thread-166 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2c8de680 to 127.0.0.1:64063 2024-12-10T12:33:55,505 DEBUG [Thread-160 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x10c964e8 to 127.0.0.1:64063 2024-12-10T12:33:55,505 DEBUG [Thread-164 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x527c6d40 to 127.0.0.1:64063 2024-12-10T12:33:55,505 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:55,505 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:55,505 DEBUG [Thread-166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:55,505 DEBUG [Thread-164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:55,505 DEBUG [Thread-162 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x72e97e4b to 127.0.0.1:64063 2024-12-10T12:33:55,505 DEBUG [Thread-162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:55,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/343ff519af914876ba236772edf74800 2024-12-10T12:33:55,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/f6c3eba4ba0a4b5fa52fa6d620f3b34b is 50, key is test_row_0/B:col10/1733834035492/Put/seqid=0 2024-12-10T12:33:55,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741951_1127 (size=12301) 2024-12-10T12:33:56,057 DEBUG [Thread-149 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5095ba91 to 127.0.0.1:64063 2024-12-10T12:33:56,058 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:56,064 DEBUG [Thread-157 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5a8f4734 to 127.0.0.1:64063 2024-12-10T12:33:56,065 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-10T12:33:56,072 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 34 completed 2024-12-10T12:33:56,076 DEBUG [Thread-153 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62c43377 to 127.0.0.1:64063 2024-12-10T12:33:56,076 DEBUG [Thread-153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:56,091 DEBUG [Thread-155 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x04977266 to 127.0.0.1:64063 2024-12-10T12:33:56,091 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 228 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 8271 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 7650 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3462 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10385 rows 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3440 2024-12-10T12:33:56,091 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10320 rows 2024-12-10T12:33:56,091 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-10T12:33:56,091 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e67f019 to 127.0.0.1:64063 2024-12-10T12:33:56,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:33:56,094 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-12-10T12:33:56,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T12:33:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T12:33:56,103 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834036103"}]},"ts":"1733834036103"} 2024-12-10T12:33:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-10T12:33:56,104 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T12:33:56,112 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T12:33:56,113 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T12:33:56,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T12:33:56,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=646ff6269f37d6ebf1ce7cbee04a0c04, UNASSIGN}] 2024-12-10T12:33:56,118 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=38, ppid=37, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=646ff6269f37d6ebf1ce7cbee04a0c04, UNASSIGN 2024-12-10T12:33:56,119 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=38 updating hbase:meta row=646ff6269f37d6ebf1ce7cbee04a0c04, regionState=CLOSING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:33:56,120 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T12:33:56,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE; CloseRegionProcedure 646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617}] 2024-12-10T12:33:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-10T12:33:56,276 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:56,279 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(124): Close 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:56,279 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-12-10T12:33:56,281 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1681): Closing 646ff6269f37d6ebf1ce7cbee04a0c04, disabling compactions & flushes 2024-12-10T12:33:56,281 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1942): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:56,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/f6c3eba4ba0a4b5fa52fa6d620f3b34b 2024-12-10T12:33:56,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/1840a202a7314471ad39b367824e1660 is 50, key is test_row_0/C:col10/1733834035492/Put/seqid=0 2024-12-10T12:33:56,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741952_1128 (size=12301) 2024-12-10T12:33:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-10T12:33:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-10T12:33:56,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/1840a202a7314471ad39b367824e1660 2024-12-10T12:33:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/343ff519af914876ba236772edf74800 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/343ff519af914876ba236772edf74800 2024-12-10T12:33:56,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/343ff519af914876ba236772edf74800, entries=200, sequenceid=540, filesize=14.4 K 2024-12-10T12:33:56,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/f6c3eba4ba0a4b5fa52fa6d620f3b34b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/f6c3eba4ba0a4b5fa52fa6d620f3b34b 2024-12-10T12:33:56,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/f6c3eba4ba0a4b5fa52fa6d620f3b34b, entries=150, sequenceid=540, filesize=12.0 K 2024-12-10T12:33:56,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/1840a202a7314471ad39b367824e1660 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1840a202a7314471ad39b367824e1660 2024-12-10T12:33:56,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1840a202a7314471ad39b367824e1660, entries=150, sequenceid=540, filesize=12.0 K 2024-12-10T12:33:56,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=67.09 KB/68700 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 1279ms, sequenceid=540, compaction requested=true 2024-12-10T12:33:56,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:56,772 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:56,772 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:56,772 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. after waiting 0 ms 2024-12-10T12:33:56,772 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:56,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:56,772 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. because compaction request was cancelled 2024-12-10T12:33:56,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:56,772 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:A 2024-12-10T12:33:56,772 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(2837): Flushing 646ff6269f37d6ebf1ce7cbee04a0c04 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:33:56,772 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. because compaction request was cancelled 2024-12-10T12:33:56,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:B, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:56,772 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:B 2024-12-10T12:33:56,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:56,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 646ff6269f37d6ebf1ce7cbee04a0c04:C, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:33:56,772 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=A 2024-12-10T12:33:56,772 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. because compaction request was cancelled 2024-12-10T12:33:56,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:33:56,772 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 646ff6269f37d6ebf1ce7cbee04a0c04:C 2024-12-10T12:33:56,772 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:56,772 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=B 2024-12-10T12:33:56,773 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:56,773 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 646ff6269f37d6ebf1ce7cbee04a0c04, store=C 2024-12-10T12:33:56,773 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:33:56,776 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/c313fd602e2341549e9520c65cee6266 is 50, key is test_row_0/A:col10/1733834036075/Put/seqid=0 2024-12-10T12:33:56,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741953_1129 (size=12301) 2024-12-10T12:33:57,182 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/c313fd602e2341549e9520c65cee6266 2024-12-10T12:33:57,199 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/77cc0ab34c0349f4a2919bdbba439983 is 50, key is test_row_0/B:col10/1733834036075/Put/seqid=0 2024-12-10T12:33:57,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741954_1130 (size=12301) 2024-12-10T12:33:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-10T12:33:57,606 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/77cc0ab34c0349f4a2919bdbba439983 2024-12-10T12:33:57,617 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/4b2be5f17c994308adf320764a649428 is 50, key is test_row_0/C:col10/1733834036075/Put/seqid=0 2024-12-10T12:33:57,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741955_1131 (size=12301) 2024-12-10T12:33:58,024 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/4b2be5f17c994308adf320764a649428 2024-12-10T12:33:58,034 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/A/c313fd602e2341549e9520c65cee6266 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/c313fd602e2341549e9520c65cee6266 2024-12-10T12:33:58,040 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/c313fd602e2341549e9520c65cee6266, entries=150, sequenceid=553, filesize=12.0 K 2024-12-10T12:33:58,041 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/B/77cc0ab34c0349f4a2919bdbba439983 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/77cc0ab34c0349f4a2919bdbba439983 2024-12-10T12:33:58,046 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/77cc0ab34c0349f4a2919bdbba439983, entries=150, sequenceid=553, filesize=12.0 K 2024-12-10T12:33:58,047 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/.tmp/C/4b2be5f17c994308adf320764a649428 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4b2be5f17c994308adf320764a649428 2024-12-10T12:33:58,051 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4b2be5f17c994308adf320764a649428, entries=150, sequenceid=553, filesize=12.0 K 2024-12-10T12:33:58,051 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 646ff6269f37d6ebf1ce7cbee04a0c04 in 1279ms, sequenceid=553, compaction requested=true 2024-12-10T12:33:58,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/93a89651f3014d8eacf0d57e9421c8d9, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/703b85bc7fa94857a0529a840f00460a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/db510d7bc7a545a9b215579dfbecdcf5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a6be2a304b6944179c0189836c0548a0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/73b539741ca14e48b573e00b6c4f1dad, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/500ecf0e0ff1429f8ae823b8db3763ee, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5f2b3b51540e4154806cf3fbdd35c5a5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d49f4d9cb9944dbcae344b76d3ef3c01, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b70d910e71704dd0a12d3b0488438eee, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/59ca3ebb6c9448b3b1879b387f08eeef, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/8a8d3e6748bd4310910980a2a3342993, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d88ef7818e6c49ec8416260858064c59, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a9e8eaa8d5c24024b6c1713798a8aaf3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a230489259fe47fa9a51661cec85ada5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/60d12d476a5c4f0fa918bb2611c38265, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5c23341d38724c8b8fd6ef3ae13d33b8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/de303161e20b449c84a3f5a99fe4fe76, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/106738bba63d4281bb5b02f413ddc163, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2a9b158c2ca4473aa38234e84f24bf21, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/43f72a9830f24e0280c0d8d03c7ced7a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/cc9aad7b672e417fac8251568a2f2a50, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/f9f5e44900a74e1b841d53cff354999f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/1094b09ddb084655bfa49f64aed3265c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/7096fb8a217842cf9fe3821319d30f2d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/840598c7b18b4951874a6aa04a6a7f72, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/89f8658e86d94e2ea08eac8989a72e4c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d2de35c525154739a14caa7d533739e7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/0ea80d13985c48f1a3eaa44df5440b84, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b5163dcf45bd4e2d9b6e236633db5fb1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/26186e47287d4cf08c74875aa9270286, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/97f8d420439a4aaaa3d9a018550ae43e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/fb5f61360b3b47c38c4afb31ff0f2119, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2365b44310c14e268050038db7bbeb5f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/6895cf7e9dd94dd8bda98a5b7bd8b7da, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/48ecc529d80145b49325e5197f973823] to archive 2024-12-10T12:33:58,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:33:58,061 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/93a89651f3014d8eacf0d57e9421c8d9 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/93a89651f3014d8eacf0d57e9421c8d9 2024-12-10T12:33:58,061 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5f2b3b51540e4154806cf3fbdd35c5a5 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5f2b3b51540e4154806cf3fbdd35c5a5 2024-12-10T12:33:58,062 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a6be2a304b6944179c0189836c0548a0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a6be2a304b6944179c0189836c0548a0 2024-12-10T12:33:58,062 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/703b85bc7fa94857a0529a840f00460a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/703b85bc7fa94857a0529a840f00460a 2024-12-10T12:33:58,062 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/db510d7bc7a545a9b215579dfbecdcf5 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/db510d7bc7a545a9b215579dfbecdcf5 2024-12-10T12:33:58,062 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/500ecf0e0ff1429f8ae823b8db3763ee to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/500ecf0e0ff1429f8ae823b8db3763ee 2024-12-10T12:33:58,062 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/73b539741ca14e48b573e00b6c4f1dad to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/73b539741ca14e48b573e00b6c4f1dad 2024-12-10T12:33:58,063 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d49f4d9cb9944dbcae344b76d3ef3c01 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d49f4d9cb9944dbcae344b76d3ef3c01 2024-12-10T12:33:58,064 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b70d910e71704dd0a12d3b0488438eee to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b70d910e71704dd0a12d3b0488438eee 2024-12-10T12:33:58,064 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/59ca3ebb6c9448b3b1879b387f08eeef to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/59ca3ebb6c9448b3b1879b387f08eeef 2024-12-10T12:33:58,065 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/8a8d3e6748bd4310910980a2a3342993 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/8a8d3e6748bd4310910980a2a3342993 2024-12-10T12:33:58,065 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d88ef7818e6c49ec8416260858064c59 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d88ef7818e6c49ec8416260858064c59 2024-12-10T12:33:58,065 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/60d12d476a5c4f0fa918bb2611c38265 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/60d12d476a5c4f0fa918bb2611c38265 2024-12-10T12:33:58,065 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a9e8eaa8d5c24024b6c1713798a8aaf3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a9e8eaa8d5c24024b6c1713798a8aaf3 2024-12-10T12:33:58,066 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5c23341d38724c8b8fd6ef3ae13d33b8 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/5c23341d38724c8b8fd6ef3ae13d33b8 2024-12-10T12:33:58,066 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a230489259fe47fa9a51661cec85ada5 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/a230489259fe47fa9a51661cec85ada5 2024-12-10T12:33:58,067 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/de303161e20b449c84a3f5a99fe4fe76 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/de303161e20b449c84a3f5a99fe4fe76 2024-12-10T12:33:58,067 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/43f72a9830f24e0280c0d8d03c7ced7a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/43f72a9830f24e0280c0d8d03c7ced7a 2024-12-10T12:33:58,067 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/cc9aad7b672e417fac8251568a2f2a50 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/cc9aad7b672e417fac8251568a2f2a50 2024-12-10T12:33:58,068 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/106738bba63d4281bb5b02f413ddc163 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/106738bba63d4281bb5b02f413ddc163 2024-12-10T12:33:58,068 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2a9b158c2ca4473aa38234e84f24bf21 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2a9b158c2ca4473aa38234e84f24bf21 2024-12-10T12:33:58,069 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/f9f5e44900a74e1b841d53cff354999f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/f9f5e44900a74e1b841d53cff354999f 2024-12-10T12:33:58,069 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/1094b09ddb084655bfa49f64aed3265c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/1094b09ddb084655bfa49f64aed3265c 2024-12-10T12:33:58,069 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/7096fb8a217842cf9fe3821319d30f2d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/7096fb8a217842cf9fe3821319d30f2d 2024-12-10T12:33:58,070 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/89f8658e86d94e2ea08eac8989a72e4c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/89f8658e86d94e2ea08eac8989a72e4c 2024-12-10T12:33:58,070 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d2de35c525154739a14caa7d533739e7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/d2de35c525154739a14caa7d533739e7 2024-12-10T12:33:58,070 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/840598c7b18b4951874a6aa04a6a7f72 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/840598c7b18b4951874a6aa04a6a7f72 2024-12-10T12:33:58,071 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b5163dcf45bd4e2d9b6e236633db5fb1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/b5163dcf45bd4e2d9b6e236633db5fb1 2024-12-10T12:33:58,071 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/0ea80d13985c48f1a3eaa44df5440b84 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/0ea80d13985c48f1a3eaa44df5440b84 2024-12-10T12:33:58,072 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/26186e47287d4cf08c74875aa9270286 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/26186e47287d4cf08c74875aa9270286 2024-12-10T12:33:58,072 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/97f8d420439a4aaaa3d9a018550ae43e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/97f8d420439a4aaaa3d9a018550ae43e 2024-12-10T12:33:58,072 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/fb5f61360b3b47c38c4afb31ff0f2119 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/fb5f61360b3b47c38c4afb31ff0f2119 2024-12-10T12:33:58,072 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/6895cf7e9dd94dd8bda98a5b7bd8b7da to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/6895cf7e9dd94dd8bda98a5b7bd8b7da 2024-12-10T12:33:58,072 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2365b44310c14e268050038db7bbeb5f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/2365b44310c14e268050038db7bbeb5f 2024-12-10T12:33:58,072 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/48ecc529d80145b49325e5197f973823 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/48ecc529d80145b49325e5197f973823 2024-12-10T12:33:58,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/de8e7f5ec18c458d9e2d53216da35e7a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2bb800e66edb4a0cb717d8e00f0dccc5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e5711c7133340fea324661abb1588e8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/417e9c66d2204b6799e6bac035377088, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0522d462da0b48a8be6a8dadf8cca598, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/5970875972454e45848a6412e0e56176, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a092c717d7934ef5870e54999e203702, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e2dacfccd1f436bbcac80d03469fe17, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/227afdb4838f431ea2a579eb8d096f76, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c42427fc45342f9b40d2282a2a90b92, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0e33bde1d3b442ce8933175645f26c58, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1a340ae90a234917afffb91c8161b10a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e28f5de8d55a4e638288f6ef9f03278c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a99ed42c70cd43438fa54e604c2570bc, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/ed9a9dd68b764267ac6f5294f9e8bcb3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/aefdfbae8862413d88d77b0a12d5438a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/90a37381d7ba464da129fc11ac440935, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c6dab3376b548b89528938eb0863357, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/85e685e89a0c49b2b083cecbd778ea1d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/c5ade30c2cb147cd80f43c1458ea17df, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/28855faf257f455ba0af157da06ce040, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e4e16ea2bda74d7c8f4b13c66804f630, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/11d8c17db37a401d84f00c2352ffb30f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/dc569f4db7bc4c988e839fd242611010, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/cee3d259e9774cf284558c47c27faa18, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/67dbbebf5d5943a4a26d88ced1815534, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1ad2d9ec236948bf8e6c721ff7b8423d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/79d3535e85204752bc4676e045138c0c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/eae7648dbee849388b1377c4a7b93d61, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/be0a92d5a23249c1ab7931a1c9327793, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2aa63874a6a84e3aaff9d49e9fe24263, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/74dc7d2b52e84bda9c3da902c577be97, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/19b83d29d1eb44aa8d91da824e3a02fa, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/644bf27af9f54cf5b780b6f508e46118, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/b0fee64d35fc42b39dd5667edeab5344] to archive 2024-12-10T12:33:58,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:33:58,092 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/de8e7f5ec18c458d9e2d53216da35e7a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/de8e7f5ec18c458d9e2d53216da35e7a 2024-12-10T12:33:58,092 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0522d462da0b48a8be6a8dadf8cca598 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0522d462da0b48a8be6a8dadf8cca598 2024-12-10T12:33:58,092 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/417e9c66d2204b6799e6bac035377088 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/417e9c66d2204b6799e6bac035377088 2024-12-10T12:33:58,092 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a092c717d7934ef5870e54999e203702 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a092c717d7934ef5870e54999e203702 2024-12-10T12:33:58,092 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e2dacfccd1f436bbcac80d03469fe17 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e2dacfccd1f436bbcac80d03469fe17 2024-12-10T12:33:58,092 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/5970875972454e45848a6412e0e56176 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/5970875972454e45848a6412e0e56176 2024-12-10T12:33:58,093 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e5711c7133340fea324661abb1588e8 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/4e5711c7133340fea324661abb1588e8 2024-12-10T12:33:58,094 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2bb800e66edb4a0cb717d8e00f0dccc5 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2bb800e66edb4a0cb717d8e00f0dccc5 2024-12-10T12:33:58,094 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/227afdb4838f431ea2a579eb8d096f76 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/227afdb4838f431ea2a579eb8d096f76 2024-12-10T12:33:58,094 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1a340ae90a234917afffb91c8161b10a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1a340ae90a234917afffb91c8161b10a 2024-12-10T12:33:58,094 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c42427fc45342f9b40d2282a2a90b92 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c42427fc45342f9b40d2282a2a90b92 2024-12-10T12:33:58,094 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a99ed42c70cd43438fa54e604c2570bc to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/a99ed42c70cd43438fa54e604c2570bc 2024-12-10T12:33:58,095 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e28f5de8d55a4e638288f6ef9f03278c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e28f5de8d55a4e638288f6ef9f03278c 2024-12-10T12:33:58,095 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0e33bde1d3b442ce8933175645f26c58 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/0e33bde1d3b442ce8933175645f26c58 2024-12-10T12:33:58,095 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/ed9a9dd68b764267ac6f5294f9e8bcb3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/ed9a9dd68b764267ac6f5294f9e8bcb3 2024-12-10T12:33:58,096 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/aefdfbae8862413d88d77b0a12d5438a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/aefdfbae8862413d88d77b0a12d5438a 2024-12-10T12:33:58,096 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/90a37381d7ba464da129fc11ac440935 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/90a37381d7ba464da129fc11ac440935 2024-12-10T12:33:58,096 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c6dab3376b548b89528938eb0863357 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/3c6dab3376b548b89528938eb0863357 2024-12-10T12:33:58,097 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e4e16ea2bda74d7c8f4b13c66804f630 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/e4e16ea2bda74d7c8f4b13c66804f630 2024-12-10T12:33:58,097 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/28855faf257f455ba0af157da06ce040 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/28855faf257f455ba0af157da06ce040 2024-12-10T12:33:58,097 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/c5ade30c2cb147cd80f43c1458ea17df to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/c5ade30c2cb147cd80f43c1458ea17df 2024-12-10T12:33:58,097 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/85e685e89a0c49b2b083cecbd778ea1d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/85e685e89a0c49b2b083cecbd778ea1d 2024-12-10T12:33:58,097 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/11d8c17db37a401d84f00c2352ffb30f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/11d8c17db37a401d84f00c2352ffb30f 2024-12-10T12:33:58,098 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1ad2d9ec236948bf8e6c721ff7b8423d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/1ad2d9ec236948bf8e6c721ff7b8423d 2024-12-10T12:33:58,098 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/dc569f4db7bc4c988e839fd242611010 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/dc569f4db7bc4c988e839fd242611010 2024-12-10T12:33:58,099 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/cee3d259e9774cf284558c47c27faa18 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/cee3d259e9774cf284558c47c27faa18 2024-12-10T12:33:58,099 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/79d3535e85204752bc4676e045138c0c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/79d3535e85204752bc4676e045138c0c 2024-12-10T12:33:58,099 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/eae7648dbee849388b1377c4a7b93d61 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/eae7648dbee849388b1377c4a7b93d61 2024-12-10T12:33:58,100 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/67dbbebf5d5943a4a26d88ced1815534 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/67dbbebf5d5943a4a26d88ced1815534 2024-12-10T12:33:58,101 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/19b83d29d1eb44aa8d91da824e3a02fa to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/19b83d29d1eb44aa8d91da824e3a02fa 2024-12-10T12:33:58,101 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2aa63874a6a84e3aaff9d49e9fe24263 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/2aa63874a6a84e3aaff9d49e9fe24263 2024-12-10T12:33:58,101 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/74dc7d2b52e84bda9c3da902c577be97 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/74dc7d2b52e84bda9c3da902c577be97 2024-12-10T12:33:58,101 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/644bf27af9f54cf5b780b6f508e46118 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/644bf27af9f54cf5b780b6f508e46118 2024-12-10T12:33:58,101 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/b0fee64d35fc42b39dd5667edeab5344 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/b0fee64d35fc42b39dd5667edeab5344 2024-12-10T12:33:58,101 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/be0a92d5a23249c1ab7931a1c9327793 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/be0a92d5a23249c1ab7931a1c9327793 2024-12-10T12:33:58,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/01578f0f6c244cd3a182950b4d89d810, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4820e4b5874b49e3a54ff74dfcacf325, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8b54544aa137446a8375f9bc80e03f5a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/879e8b35efa74e2ea2553217f0caf07f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/43bc1bfc3e2a4f3a9e6536200e274f8b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3e4fdbf06a634be0bad28e48fe089559, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/242e19ab373f4b1f992f1b4a7220428a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8ec2e2d4ef0a4d408b5180198ddf0b0a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ead4cd0427d94ece92a391defd59b7f3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f42a6b8a37344724a345e01211ef3f8a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/80d1b99fb64e4c9cb6e8fa7c58b54750, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8bc76b39f0954771bafe0d49aaae4644, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/27066b2e17ba407581c41940aafcd4fd, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6757e4e228b4416886dfbb448233f75a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f5f4f56b3bc4491cae07e9d4ee5b9044, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2579e32046b343eb9b28611a62eb971d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aa205571e4644d07884717572c1ab1bb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a43df02088f946b29998fe55ad6e4c9d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/18ec49454ca04535997c6faad4872349, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/9fd6a73067304b6ea685e2f6aa7d37e2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1b907b4ff2694afe9756fd187e80c516, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/48a04c42fea546b2a225c3abde793d2e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aad7458921fa4981a812c0cd0a116814, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/fa8db5cef9b54fc7ac0ea7293a30007b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2f885bab09e845a9b8513a9760c791f3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/d379d8caddb447c9987e3430f6d38719, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/70e4c6d99abd4fa792d0fe79ddde6184, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5a3fd308fe7a4a64ababa60fb7d20e76, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/28471967f1454463bc27b69bbde4afa0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/bac44ed3099b44e0989e579c5d83a806, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3b5a93e369454adf8fea226aef7bf81a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a4bfd8fc7f0a4dcba32ec5660c3772c2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6bbcd6b7d3334effad7c076c07dc1296, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ca1bbcf4b39444d2af697bec7d76eab0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5f5ae41f78924e569cc444f9792e4229] to archive 2024-12-10T12:33:58,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:33:58,106 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/879e8b35efa74e2ea2553217f0caf07f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/879e8b35efa74e2ea2553217f0caf07f 2024-12-10T12:33:58,106 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8b54544aa137446a8375f9bc80e03f5a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8b54544aa137446a8375f9bc80e03f5a 2024-12-10T12:33:58,107 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/01578f0f6c244cd3a182950b4d89d810 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/01578f0f6c244cd3a182950b4d89d810 2024-12-10T12:33:58,107 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8ec2e2d4ef0a4d408b5180198ddf0b0a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8ec2e2d4ef0a4d408b5180198ddf0b0a 2024-12-10T12:33:58,107 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/43bc1bfc3e2a4f3a9e6536200e274f8b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/43bc1bfc3e2a4f3a9e6536200e274f8b 2024-12-10T12:33:58,107 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3e4fdbf06a634be0bad28e48fe089559 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3e4fdbf06a634be0bad28e48fe089559 2024-12-10T12:33:58,108 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/242e19ab373f4b1f992f1b4a7220428a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/242e19ab373f4b1f992f1b4a7220428a 2024-12-10T12:33:58,108 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4820e4b5874b49e3a54ff74dfcacf325 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4820e4b5874b49e3a54ff74dfcacf325 2024-12-10T12:33:58,109 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8bc76b39f0954771bafe0d49aaae4644 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/8bc76b39f0954771bafe0d49aaae4644 2024-12-10T12:33:58,109 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ead4cd0427d94ece92a391defd59b7f3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ead4cd0427d94ece92a391defd59b7f3 2024-12-10T12:33:58,109 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6757e4e228b4416886dfbb448233f75a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6757e4e228b4416886dfbb448233f75a 2024-12-10T12:33:58,109 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f42a6b8a37344724a345e01211ef3f8a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f42a6b8a37344724a345e01211ef3f8a 2024-12-10T12:33:58,109 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/27066b2e17ba407581c41940aafcd4fd to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/27066b2e17ba407581c41940aafcd4fd 2024-12-10T12:33:58,110 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/80d1b99fb64e4c9cb6e8fa7c58b54750 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/80d1b99fb64e4c9cb6e8fa7c58b54750 2024-12-10T12:33:58,110 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f5f4f56b3bc4491cae07e9d4ee5b9044 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/f5f4f56b3bc4491cae07e9d4ee5b9044 2024-12-10T12:33:58,111 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2579e32046b343eb9b28611a62eb971d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2579e32046b343eb9b28611a62eb971d 2024-12-10T12:33:58,112 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aa205571e4644d07884717572c1ab1bb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aa205571e4644d07884717572c1ab1bb 2024-12-10T12:33:58,112 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/18ec49454ca04535997c6faad4872349 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/18ec49454ca04535997c6faad4872349 2024-12-10T12:33:58,112 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a43df02088f946b29998fe55ad6e4c9d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a43df02088f946b29998fe55ad6e4c9d 2024-12-10T12:33:58,112 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1b907b4ff2694afe9756fd187e80c516 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1b907b4ff2694afe9756fd187e80c516 2024-12-10T12:33:58,112 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/9fd6a73067304b6ea685e2f6aa7d37e2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/9fd6a73067304b6ea685e2f6aa7d37e2 2024-12-10T12:33:58,112 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aad7458921fa4981a812c0cd0a116814 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/aad7458921fa4981a812c0cd0a116814 2024-12-10T12:33:58,113 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/48a04c42fea546b2a225c3abde793d2e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/48a04c42fea546b2a225c3abde793d2e 2024-12-10T12:33:58,113 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/fa8db5cef9b54fc7ac0ea7293a30007b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/fa8db5cef9b54fc7ac0ea7293a30007b 2024-12-10T12:33:58,114 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2f885bab09e845a9b8513a9760c791f3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/2f885bab09e845a9b8513a9760c791f3 2024-12-10T12:33:58,114 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/d379d8caddb447c9987e3430f6d38719 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/d379d8caddb447c9987e3430f6d38719 2024-12-10T12:33:58,114 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/70e4c6d99abd4fa792d0fe79ddde6184 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/70e4c6d99abd4fa792d0fe79ddde6184 2024-12-10T12:33:58,115 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5a3fd308fe7a4a64ababa60fb7d20e76 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5a3fd308fe7a4a64ababa60fb7d20e76 2024-12-10T12:33:58,115 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/28471967f1454463bc27b69bbde4afa0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/28471967f1454463bc27b69bbde4afa0 2024-12-10T12:33:58,115 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3b5a93e369454adf8fea226aef7bf81a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/3b5a93e369454adf8fea226aef7bf81a 2024-12-10T12:33:58,115 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/bac44ed3099b44e0989e579c5d83a806 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/bac44ed3099b44e0989e579c5d83a806 2024-12-10T12:33:58,116 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a4bfd8fc7f0a4dcba32ec5660c3772c2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/a4bfd8fc7f0a4dcba32ec5660c3772c2 2024-12-10T12:33:58,116 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5f5ae41f78924e569cc444f9792e4229 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/5f5ae41f78924e569cc444f9792e4229 2024-12-10T12:33:58,116 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6bbcd6b7d3334effad7c076c07dc1296 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/6bbcd6b7d3334effad7c076c07dc1296 2024-12-10T12:33:58,116 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ca1bbcf4b39444d2af697bec7d76eab0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/ca1bbcf4b39444d2af697bec7d76eab0 2024-12-10T12:33:58,120 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/recovered.edits/556.seqid, newMaxSeqId=556, maxSeqId=1 2024-12-10T12:33:58,123 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04. 2024-12-10T12:33:58,123 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1635): Region close journal for 646ff6269f37d6ebf1ce7cbee04a0c04: 2024-12-10T12:33:58,124 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(170): Closed 646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:58,125 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=38 updating hbase:meta row=646ff6269f37d6ebf1ce7cbee04a0c04, regionState=CLOSED 2024-12-10T12:33:58,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=39, resume processing ppid=38 2024-12-10T12:33:58,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, ppid=38, state=SUCCESS; CloseRegionProcedure 646ff6269f37d6ebf1ce7cbee04a0c04, server=fad905551524,38809,1733834007617 in 2.0060 sec 2024-12-10T12:33:58,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-12-10T12:33:58,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=646ff6269f37d6ebf1ce7cbee04a0c04, UNASSIGN in 2.0090 sec 2024-12-10T12:33:58,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-12-10T12:33:58,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0160 sec 2024-12-10T12:33:58,132 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834038132"}]},"ts":"1733834038132"} 2024-12-10T12:33:58,133 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T12:33:58,175 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T12:33:58,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 2.0770 sec 2024-12-10T12:33:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-10T12:33:58,210 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 36 completed 2024-12-10T12:33:58,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T12:33:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=40, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:33:58,219 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=40, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:33:58,220 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=40, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:33:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-12-10T12:33:58,222 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:58,226 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/recovered.edits] 2024-12-10T12:33:58,230 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/645a52c0020249288cc97e6dd534f5a7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/645a52c0020249288cc97e6dd534f5a7 2024-12-10T12:33:58,230 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/343ff519af914876ba236772edf74800 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/343ff519af914876ba236772edf74800 2024-12-10T12:33:58,230 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/c313fd602e2341549e9520c65cee6266 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/c313fd602e2341549e9520c65cee6266 2024-12-10T12:33:58,230 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/7705315176964d8c9545ebd8a799c900 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/A/7705315176964d8c9545ebd8a799c900 2024-12-10T12:33:58,234 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/77cc0ab34c0349f4a2919bdbba439983 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/77cc0ab34c0349f4a2919bdbba439983 2024-12-10T12:33:58,234 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/76caccdb5e3d4653a36e990023290473 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/76caccdb5e3d4653a36e990023290473 2024-12-10T12:33:58,234 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/f6c3eba4ba0a4b5fa52fa6d620f3b34b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/f6c3eba4ba0a4b5fa52fa6d620f3b34b 2024-12-10T12:33:58,234 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/9d1fbb98a6a5426bac2e54b9b72e7258 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/B/9d1fbb98a6a5426bac2e54b9b72e7258 2024-12-10T12:33:58,238 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/82e58d7869b84ce6be2cada0d624fecb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/82e58d7869b84ce6be2cada0d624fecb 2024-12-10T12:33:58,238 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1840a202a7314471ad39b367824e1660 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/1840a202a7314471ad39b367824e1660 2024-12-10T12:33:58,238 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/7825cd03c0a64961a3d5f0b071716bf6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/7825cd03c0a64961a3d5f0b071716bf6 2024-12-10T12:33:58,238 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4b2be5f17c994308adf320764a649428 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/C/4b2be5f17c994308adf320764a649428 2024-12-10T12:33:58,241 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/recovered.edits/556.seqid to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04/recovered.edits/556.seqid 2024-12-10T12:33:58,241 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/646ff6269f37d6ebf1ce7cbee04a0c04 2024-12-10T12:33:58,241 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T12:33:58,246 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=40, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:33:58,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-10T12:33:58,252 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T12:33:58,283 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T12:33:58,284 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=40, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:33:58,284 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T12:33:58,285 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733834038284"}]},"ts":"9223372036854775807"} 2024-12-10T12:33:58,288 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-12-10T12:33:58,288 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 646ff6269f37d6ebf1ce7cbee04a0c04, NAME => 'TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T12:33:58,288 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T12:33:58,288 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733834038288"}]},"ts":"9223372036854775807"} 2024-12-10T12:33:58,291 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-12-10T12:33:58,301 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=40, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:33:58,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-12-10T12:33:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-12-10T12:33:58,322 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 40 completed 2024-12-10T12:33:58,335 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMixedAtomicity Thread=247 (was 219) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-193056614_22 at /127.0.0.1:56362 [Waiting for operation #268] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x31d7b3b1-shared-pool-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fad905551524:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x31d7b3b1-shared-pool-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fad905551524:38809-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-282036948_22 at /127.0.0.1:40218 [Waiting for operation #254] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fad905551524:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x31d7b3b1-shared-pool-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-193056614_22 at /127.0.0.1:40322 [Waiting for operation #223] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x31d7b3b1-shared-pool-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=460 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 130) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9603 (was 10166) 2024-12-10T12:33:58,343 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=460, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=9602 2024-12-10T12:33:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T12:33:58,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:33:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=41, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T12:33:58,346 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=41, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T12:33:58,346 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:58,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 41 2024-12-10T12:33:58,347 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=41, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T12:33:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-10T12:33:58,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741956_1132 (size=963) 2024-12-10T12:33:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-10T12:33:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-10T12:33:58,759 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 2024-12-10T12:33:58,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741957_1133 (size=53) 2024-12-10T12:33:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-10T12:33:59,170 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:59,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing d32afc9a2cd1759fc0fd2b0a98d70e22, disabling compactions & flushes 2024-12-10T12:33:59,171 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:33:59,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:33:59,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. after waiting 0 ms 2024-12-10T12:33:59,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:33:59,171 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:33:59,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:33:59,174 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=41, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T12:33:59,175 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733834039174"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733834039174"}]},"ts":"1733834039174"} 2024-12-10T12:33:59,178 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-10T12:33:59,179 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=41, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T12:33:59,179 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834039179"}]},"ts":"1733834039179"} 2024-12-10T12:33:59,180 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T12:33:59,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, ASSIGN}] 2024-12-10T12:33:59,201 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, ASSIGN 2024-12-10T12:33:59,202 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, ASSIGN; state=OFFLINE, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=false 2024-12-10T12:33:59,353 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=d32afc9a2cd1759fc0fd2b0a98d70e22, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:33:59,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; OpenRegionProcedure d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617}] 2024-12-10T12:33:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-10T12:33:59,510 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:33:59,517 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:33:59,518 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7285): Opening region: {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:33:59,519 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:33:59,519 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:33:59,519 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7327): checking encryption for d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:33:59,519 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7330): checking classloading for d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:33:59,522 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:33:59,524 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:33:59,525 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d32afc9a2cd1759fc0fd2b0a98d70e22 columnFamilyName A 2024-12-10T12:33:59,525 DEBUG [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:59,525 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(327): Store=d32afc9a2cd1759fc0fd2b0a98d70e22/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:59,526 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:33:59,527 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:33:59,528 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d32afc9a2cd1759fc0fd2b0a98d70e22 columnFamilyName B 2024-12-10T12:33:59,528 DEBUG [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:59,529 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(327): Store=d32afc9a2cd1759fc0fd2b0a98d70e22/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:59,529 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:33:59,530 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:33:59,530 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d32afc9a2cd1759fc0fd2b0a98d70e22 columnFamilyName C 2024-12-10T12:33:59,531 DEBUG [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:33:59,531 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(327): Store=d32afc9a2cd1759fc0fd2b0a98d70e22/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:33:59,531 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:33:59,532 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:33:59,532 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:33:59,534 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:33:59,535 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1085): writing seq id for d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:33:59,538 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:33:59,538 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1102): Opened d32afc9a2cd1759fc0fd2b0a98d70e22; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71254923, jitterRate=0.06178109347820282}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:33:59,539 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1001): Region open journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:33:59,540 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., pid=43, masterSystemTime=1733834039509 2024-12-10T12:33:59,541 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:33:59,541 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:33:59,542 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=d32afc9a2cd1759fc0fd2b0a98d70e22, regionState=OPEN, openSeqNum=2, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:33:59,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-12-10T12:33:59,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; OpenRegionProcedure d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 in 188 msec 2024-12-10T12:33:59,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=42, resume processing ppid=41 2024-12-10T12:33:59,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, ppid=41, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, ASSIGN in 345 msec 2024-12-10T12:33:59,547 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=41, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T12:33:59,547 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834039547"}]},"ts":"1733834039547"} 2024-12-10T12:33:59,548 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T12:33:59,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=41, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T12:33:59,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2140 sec 2024-12-10T12:34:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-10T12:34:00,459 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 41 completed 2024-12-10T12:34:00,463 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7cca453a to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@350b322d 2024-12-10T12:34:00,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26401a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:00,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:00,512 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43270, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:00,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T12:34:00,516 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35156, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T12:34:00,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T12:34:00,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$14(2798): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:34:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=44, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T12:34:00,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741958_1134 (size=999) 2024-12-10T12:34:00,942 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.963 2024-12-10T12:34:00,942 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.999 2024-12-10T12:34:00,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T12:34:00,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, REOPEN/MOVE}] 2024-12-10T12:34:00,957 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, REOPEN/MOVE 2024-12-10T12:34:00,958 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=d32afc9a2cd1759fc0fd2b0a98d70e22, regionState=CLOSING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:34:00,959 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T12:34:00,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; CloseRegionProcedure d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617}] 2024-12-10T12:34:01,111 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:01,112 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(124): Close d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,112 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-12-10T12:34:01,112 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1681): Closing d32afc9a2cd1759fc0fd2b0a98d70e22, disabling compactions & flushes 2024-12-10T12:34:01,113 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,113 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,113 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. after waiting 0 ms 2024-12-10T12:34:01,113 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,122 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T12:34:01,124 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,124 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1635): Region close journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:01,124 WARN [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionServer(3786): Not adding moved region record: d32afc9a2cd1759fc0fd2b0a98d70e22 to self. 2024-12-10T12:34:01,126 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(170): Closed d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,126 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=d32afc9a2cd1759fc0fd2b0a98d70e22, regionState=CLOSED 2024-12-10T12:34:01,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-12-10T12:34:01,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; CloseRegionProcedure d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 in 169 msec 2024-12-10T12:34:01,131 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, REOPEN/MOVE; state=CLOSED, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=true 2024-12-10T12:34:01,282 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=d32afc9a2cd1759fc0fd2b0a98d70e22, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:34:01,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=48, ppid=46, state=RUNNABLE; OpenRegionProcedure d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617}] 2024-12-10T12:34:01,439 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:01,447 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,448 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7285): Opening region: {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:34:01,448 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,449 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:34:01,449 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7327): checking encryption for d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,449 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7330): checking classloading for d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,453 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,453 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:34:01,458 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d32afc9a2cd1759fc0fd2b0a98d70e22 columnFamilyName A 2024-12-10T12:34:01,460 DEBUG [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:01,461 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(327): Store=d32afc9a2cd1759fc0fd2b0a98d70e22/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:34:01,462 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,462 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:34:01,463 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d32afc9a2cd1759fc0fd2b0a98d70e22 columnFamilyName B 2024-12-10T12:34:01,463 DEBUG [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:01,463 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(327): Store=d32afc9a2cd1759fc0fd2b0a98d70e22/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:34:01,463 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,464 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:34:01,464 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d32afc9a2cd1759fc0fd2b0a98d70e22 columnFamilyName C 2024-12-10T12:34:01,464 DEBUG [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:01,465 INFO [StoreOpener-d32afc9a2cd1759fc0fd2b0a98d70e22-1 {}] regionserver.HStore(327): Store=d32afc9a2cd1759fc0fd2b0a98d70e22/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:34:01,465 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,465 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,466 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,468 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:34:01,469 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1085): writing seq id for d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,470 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1102): Opened d32afc9a2cd1759fc0fd2b0a98d70e22; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59251403, jitterRate=-0.1170852929353714}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:34:01,470 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1001): Region open journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:01,471 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., pid=48, masterSystemTime=1733834041439 2024-12-10T12:34:01,473 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,473 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,473 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=d32afc9a2cd1759fc0fd2b0a98d70e22, regionState=OPEN, openSeqNum=5, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:34:01,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=48, resume processing ppid=46 2024-12-10T12:34:01,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, ppid=46, state=SUCCESS; OpenRegionProcedure d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 in 189 msec 2024-12-10T12:34:01,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-12-10T12:34:01,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, REOPEN/MOVE in 519 msec 2024-12-10T12:34:01,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=45, resume processing ppid=44 2024-12-10T12:34:01,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, ppid=44, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-12-10T12:34:01,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-12-10T12:34:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-12-10T12:34:01,488 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x433e2b26 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7bad2e85 2024-12-10T12:34:01,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491ea2ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:01,552 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x176c5c1b to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@328f994d 2024-12-10T12:34:01,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b44b1e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:01,571 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x24f64590 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@19a533a3 2024-12-10T12:34:01,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e904d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:01,583 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7c5c4716 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@465dc764 2024-12-10T12:34:01,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885d2d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:01,595 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68f0be85 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@247c0c93 2024-12-10T12:34:01,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e911df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:01,607 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x517ff977 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b727d6e 2024-12-10T12:34:01,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c16cd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:01,619 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3448d233 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c7940d9 2024-12-10T12:34:01,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341384e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:01,632 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a11164b to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c38ee58 2024-12-10T12:34:01,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b120d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:01,644 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08a7e1dd to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@630684bf 2024-12-10T12:34:01,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c1ec7ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:01,661 DEBUG [hconnection-0x507aa03d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:01,661 DEBUG [hconnection-0x12651e3d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:01,662 DEBUG [hconnection-0x736a24b5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:01,662 DEBUG [hconnection-0x48a4b892-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:01,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:01,663 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43276, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:01,663 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43274, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:01,663 DEBUG [hconnection-0x60f1f0c8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:01,663 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43302, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:01,664 DEBUG [hconnection-0x17d388d9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=49, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees 2024-12-10T12:34:01,664 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43304, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:01,665 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=49, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:01,665 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43320, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-10T12:34:01,666 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=49, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:01,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:01,667 DEBUG [hconnection-0x2d6ebeeb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:01,667 DEBUG [hconnection-0x2a388541-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:01,668 DEBUG [hconnection-0x63228597-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:01,669 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43356, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:01,669 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43340, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:01,669 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43338, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:01,669 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43330, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:01,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:34:01,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:01,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:01,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:01,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:01,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:01,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:01,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121076b23084c52344b6ad78aef5d5b9bc83_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834041679/Put/seqid=0 2024-12-10T12:34:01,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834101726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834101730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834101736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834101738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834101739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741959_1135 (size=12154) 2024-12-10T12:34:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-10T12:34:01,818 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:01,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-12-10T12:34:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:01,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:01,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834101839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834101842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834101846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834101846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834101847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-10T12:34:01,974 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:01,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-12-10T12:34:01,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:01,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:01,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:01,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834102045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834102046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834102051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834102052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834102052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,126 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:02,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-12-10T12:34:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:02,164 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121076b23084c52344b6ad78aef5d5b9bc83_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121076b23084c52344b6ad78aef5d5b9bc83_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:02,165 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/8c1d4f89020f4cc8b7fb3056b50e9c93, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:02,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/8c1d4f89020f4cc8b7fb3056b50e9c93 is 175, key is test_row_0/A:col10/1733834041679/Put/seqid=0 2024-12-10T12:34:02,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741960_1136 (size=30955) 2024-12-10T12:34:02,190 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/8c1d4f89020f4cc8b7fb3056b50e9c93 2024-12-10T12:34:02,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/a80d3d51978943638b8975b805f5a3bf is 50, key is test_row_0/B:col10/1733834041679/Put/seqid=0 2024-12-10T12:34:02,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741961_1137 (size=12001) 2024-12-10T12:34:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-10T12:34:02,279 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:02,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-12-10T12:34:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834102349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834102349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834102356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834102358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834102359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,433 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:02,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-12-10T12:34:02,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:02,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,587 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:02,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-12-10T12:34:02,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:02,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:02,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/a80d3d51978943638b8975b805f5a3bf 2024-12-10T12:34:02,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/1627927ff52a4e06aee0efa2d1bd06d7 is 50, key is test_row_0/C:col10/1733834041679/Put/seqid=0 2024-12-10T12:34:02,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741962_1138 (size=12001) 2024-12-10T12:34:02,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/1627927ff52a4e06aee0efa2d1bd06d7 2024-12-10T12:34:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/8c1d4f89020f4cc8b7fb3056b50e9c93 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8c1d4f89020f4cc8b7fb3056b50e9c93 2024-12-10T12:34:02,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8c1d4f89020f4cc8b7fb3056b50e9c93, entries=150, sequenceid=16, filesize=30.2 K 2024-12-10T12:34:02,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/a80d3d51978943638b8975b805f5a3bf as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/a80d3d51978943638b8975b805f5a3bf 2024-12-10T12:34:02,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/a80d3d51978943638b8975b805f5a3bf, entries=150, sequenceid=16, filesize=11.7 K 2024-12-10T12:34:02,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/1627927ff52a4e06aee0efa2d1bd06d7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/1627927ff52a4e06aee0efa2d1bd06d7 2024-12-10T12:34:02,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/1627927ff52a4e06aee0efa2d1bd06d7, entries=150, sequenceid=16, filesize=11.7 K 2024-12-10T12:34:02,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1031ms, sequenceid=16, compaction requested=false 2024-12-10T12:34:02,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:02,739 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:02,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-12-10T12:34:02,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:34:02,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:02,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121068d8215d6d084d0e86ee4be6baab091b_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834041720/Put/seqid=0 2024-12-10T12:34:02,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741963_1139 (size=12154) 2024-12-10T12:34:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-10T12:34:02,777 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121068d8215d6d084d0e86ee4be6baab091b_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121068d8215d6d084d0e86ee4be6baab091b_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:02,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/2e0205ccf70a47329b81d443e008148f, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:02,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/2e0205ccf70a47329b81d443e008148f is 175, key is test_row_0/A:col10/1733834041720/Put/seqid=0 2024-12-10T12:34:02,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741964_1140 (size=30955) 2024-12-10T12:34:02,798 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T12:34:02,798 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/2e0205ccf70a47329b81d443e008148f 2024-12-10T12:34:02,800 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35170, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T12:34:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/204a1483ef604b7d90b26a860e142c6c is 50, key is test_row_0/B:col10/1733834041720/Put/seqid=0 2024-12-10T12:34:02,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741965_1141 (size=12001) 2024-12-10T12:34:02,855 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/204a1483ef604b7d90b26a860e142c6c 2024-12-10T12:34:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:02,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:02,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/5e6fcd0092064fb9a8a2ef8c26cc5f4a is 50, key is test_row_0/C:col10/1733834041720/Put/seqid=0 2024-12-10T12:34:02,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834102873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834102873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834102874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834102878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741966_1142 (size=12001) 2024-12-10T12:34:02,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834102880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:02,886 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/5e6fcd0092064fb9a8a2ef8c26cc5f4a 2024-12-10T12:34:02,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/2e0205ccf70a47329b81d443e008148f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2e0205ccf70a47329b81d443e008148f 2024-12-10T12:34:02,899 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2e0205ccf70a47329b81d443e008148f, entries=150, sequenceid=41, filesize=30.2 K 2024-12-10T12:34:02,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/204a1483ef604b7d90b26a860e142c6c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/204a1483ef604b7d90b26a860e142c6c 2024-12-10T12:34:02,906 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/204a1483ef604b7d90b26a860e142c6c, entries=150, sequenceid=41, filesize=11.7 K 2024-12-10T12:34:02,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/5e6fcd0092064fb9a8a2ef8c26cc5f4a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5e6fcd0092064fb9a8a2ef8c26cc5f4a 2024-12-10T12:34:02,914 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5e6fcd0092064fb9a8a2ef8c26cc5f4a, entries=150, sequenceid=41, filesize=11.7 K 2024-12-10T12:34:02,916 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 175ms, sequenceid=41, compaction requested=false 2024-12-10T12:34:02,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:02,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:02,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-12-10T12:34:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=50 2024-12-10T12:34:02,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-12-10T12:34:02,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2520 sec 2024-12-10T12:34:02,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees in 1.2570 sec 2024-12-10T12:34:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:02,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:34:02,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:02,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:02,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:02,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:02,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:02,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:03,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121036cf9e05615045cd93782e5ab69beeba_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834042988/Put/seqid=0 2024-12-10T12:34:03,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741967_1143 (size=17034) 2024-12-10T12:34:03,018 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:03,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121036cf9e05615045cd93782e5ab69beeba_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121036cf9e05615045cd93782e5ab69beeba_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:03,027 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f366a8d71338428cb71d9603206fdbf9, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:03,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f366a8d71338428cb71d9603206fdbf9 is 175, key is test_row_0/A:col10/1733834042988/Put/seqid=0 2024-12-10T12:34:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741968_1144 (size=48139) 2024-12-10T12:34:03,042 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f366a8d71338428cb71d9603206fdbf9 2024-12-10T12:34:03,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834103041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834103045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834103045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834103049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834103049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/615adadbf6844ef5aeb826df4b976197 is 50, key is test_row_0/B:col10/1733834042988/Put/seqid=0 2024-12-10T12:34:03,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741969_1145 (size=12001) 2024-12-10T12:34:03,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/615adadbf6844ef5aeb826df4b976197 2024-12-10T12:34:03,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/2f52a85d63c44b1faccd7c503b6eaa6b is 50, key is test_row_0/C:col10/1733834042988/Put/seqid=0 2024-12-10T12:34:03,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741970_1146 (size=12001) 2024-12-10T12:34:03,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/2f52a85d63c44b1faccd7c503b6eaa6b 2024-12-10T12:34:03,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f366a8d71338428cb71d9603206fdbf9 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f366a8d71338428cb71d9603206fdbf9 2024-12-10T12:34:03,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f366a8d71338428cb71d9603206fdbf9, entries=250, sequenceid=56, filesize=47.0 K 2024-12-10T12:34:03,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/615adadbf6844ef5aeb826df4b976197 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/615adadbf6844ef5aeb826df4b976197 2024-12-10T12:34:03,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/615adadbf6844ef5aeb826df4b976197, entries=150, sequenceid=56, filesize=11.7 K 2024-12-10T12:34:03,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/2f52a85d63c44b1faccd7c503b6eaa6b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/2f52a85d63c44b1faccd7c503b6eaa6b 2024-12-10T12:34:03,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/2f52a85d63c44b1faccd7c503b6eaa6b, entries=150, sequenceid=56, filesize=11.7 K 2024-12-10T12:34:03,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 139ms, sequenceid=56, compaction requested=true 2024-12-10T12:34:03,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:03,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:03,129 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:03,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:03,129 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:03,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:03,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:03,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:03,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:03,131 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:03,131 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/A is initiating minor compaction (all files) 2024-12-10T12:34:03,131 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/A in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:03,131 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8c1d4f89020f4cc8b7fb3056b50e9c93, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2e0205ccf70a47329b81d443e008148f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f366a8d71338428cb71d9603206fdbf9] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=107.5 K 2024-12-10T12:34:03,131 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:03,131 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8c1d4f89020f4cc8b7fb3056b50e9c93, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2e0205ccf70a47329b81d443e008148f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f366a8d71338428cb71d9603206fdbf9] 2024-12-10T12:34:03,132 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:03,132 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/B is initiating minor compaction (all files) 2024-12-10T12:34:03,132 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/B in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:03,132 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/a80d3d51978943638b8975b805f5a3bf, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/204a1483ef604b7d90b26a860e142c6c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/615adadbf6844ef5aeb826df4b976197] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=35.2 K 2024-12-10T12:34:03,132 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8c1d4f89020f4cc8b7fb3056b50e9c93, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733834041673 2024-12-10T12:34:03,132 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting a80d3d51978943638b8975b805f5a3bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733834041673 2024-12-10T12:34:03,133 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2e0205ccf70a47329b81d443e008148f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733834041720 2024-12-10T12:34:03,133 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 204a1483ef604b7d90b26a860e142c6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733834041720 2024-12-10T12:34:03,134 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f366a8d71338428cb71d9603206fdbf9, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733834042872 2024-12-10T12:34:03,134 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 615adadbf6844ef5aeb826df4b976197, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733834042988 2024-12-10T12:34:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:03,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-10T12:34:03,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:03,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:03,162 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:03,162 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/d19a5feed7b5401895bdde4e62d2ed15 is 50, key is test_row_0/B:col10/1733834042988/Put/seqid=0 2024-12-10T12:34:03,165 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:03,174 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412100d129d63ad2e4d5fbfd716abcd0382e9_d32afc9a2cd1759fc0fd2b0a98d70e22 store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:03,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741971_1147 (size=12104) 2024-12-10T12:34:03,182 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412100d129d63ad2e4d5fbfd716abcd0382e9_d32afc9a2cd1759fc0fd2b0a98d70e22, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:03,183 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100d129d63ad2e4d5fbfd716abcd0382e9_d32afc9a2cd1759fc0fd2b0a98d70e22 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:03,189 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/d19a5feed7b5401895bdde4e62d2ed15 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d19a5feed7b5401895bdde4e62d2ed15 2024-12-10T12:34:03,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100c0d3654f7a54325820ba1359f335145_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834043153/Put/seqid=0 2024-12-10T12:34:03,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741972_1148 (size=4469) 2024-12-10T12:34:03,207 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#A#compaction#127 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:03,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741973_1149 (size=14594) 2024-12-10T12:34:03,207 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/B of d32afc9a2cd1759fc0fd2b0a98d70e22 into d19a5feed7b5401895bdde4e62d2ed15(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:03,207 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:03,208 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/B, priority=13, startTime=1733834043129; duration=0sec 2024-12-10T12:34:03,208 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:03,208 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:B 2024-12-10T12:34:03,208 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:03,209 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/b0eb84cb7301490b859f98c4d4bf9fae is 175, key is test_row_0/A:col10/1733834042988/Put/seqid=0 2024-12-10T12:34:03,212 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:03,212 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/C is initiating minor compaction (all files) 2024-12-10T12:34:03,212 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/C in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:03,212 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/1627927ff52a4e06aee0efa2d1bd06d7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5e6fcd0092064fb9a8a2ef8c26cc5f4a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/2f52a85d63c44b1faccd7c503b6eaa6b] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=35.2 K 2024-12-10T12:34:03,213 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1627927ff52a4e06aee0efa2d1bd06d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733834041673 2024-12-10T12:34:03,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834103212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834103214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834103217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,221 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5e6fcd0092064fb9a8a2ef8c26cc5f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733834041720 2024-12-10T12:34:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834103218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834103216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,222 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2f52a85d63c44b1faccd7c503b6eaa6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733834042988 2024-12-10T12:34:03,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741974_1150 (size=31058) 2024-12-10T12:34:03,237 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/b0eb84cb7301490b859f98c4d4bf9fae as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b0eb84cb7301490b859f98c4d4bf9fae 2024-12-10T12:34:03,239 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#C#compaction#129 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:03,240 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/be7c1dd5d80a40c2b767238d19bf2fcf is 50, key is test_row_0/C:col10/1733834042988/Put/seqid=0 2024-12-10T12:34:03,246 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/A of d32afc9a2cd1759fc0fd2b0a98d70e22 into b0eb84cb7301490b859f98c4d4bf9fae(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:03,246 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:03,246 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/A, priority=13, startTime=1733834043129; duration=0sec 2024-12-10T12:34:03,247 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:03,247 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:A 2024-12-10T12:34:03,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741975_1151 (size=12104) 2024-12-10T12:34:03,259 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/be7c1dd5d80a40c2b767238d19bf2fcf as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/be7c1dd5d80a40c2b767238d19bf2fcf 2024-12-10T12:34:03,264 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/C of d32afc9a2cd1759fc0fd2b0a98d70e22 into be7c1dd5d80a40c2b767238d19bf2fcf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:03,264 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:03,264 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/C, priority=13, startTime=1733834043131; duration=0sec 2024-12-10T12:34:03,265 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:03,265 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:C 2024-12-10T12:34:03,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834103319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834103320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834103323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834103323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834103324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834103522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834103522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834103527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834103527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834103528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,608 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:03,613 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100c0d3654f7a54325820ba1359f335145_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100c0d3654f7a54325820ba1359f335145_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:03,614 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/c5a12f966b384043ad3c33fa954efe9d, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:03,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/c5a12f966b384043ad3c33fa954efe9d is 175, key is test_row_0/A:col10/1733834043153/Put/seqid=0 2024-12-10T12:34:03,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741976_1152 (size=39549) 2024-12-10T12:34:03,626 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/c5a12f966b384043ad3c33fa954efe9d 2024-12-10T12:34:03,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/7d440809a4884eddb3563f4d2f10b594 is 50, key is test_row_0/B:col10/1733834043153/Put/seqid=0 2024-12-10T12:34:03,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741977_1153 (size=12001) 2024-12-10T12:34:03,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/7d440809a4884eddb3563f4d2f10b594 2024-12-10T12:34:03,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/c5fb5b4ff9214ee5a9e115fbb306dca3 is 50, key is test_row_0/C:col10/1733834043153/Put/seqid=0 2024-12-10T12:34:03,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741978_1154 (size=12001) 2024-12-10T12:34:03,686 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T12:34:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-10T12:34:03,771 INFO [Thread-667 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 49 completed 2024-12-10T12:34:03,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees 2024-12-10T12:34:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-10T12:34:03,774 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=51, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:03,775 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=51, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:03,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:03,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834103826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834103828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834103830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834103832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834103832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-10T12:34:03,927 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:03,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=52 2024-12-10T12:34:03,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:03,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:03,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:03,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] handler.RSProcedureHandler(58): pid=52 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:03,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=52 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=52 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:04,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/c5fb5b4ff9214ee5a9e115fbb306dca3 2024-12-10T12:34:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-10T12:34:04,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/c5a12f966b384043ad3c33fa954efe9d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c5a12f966b384043ad3c33fa954efe9d 2024-12-10T12:34:04,081 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:04,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c5a12f966b384043ad3c33fa954efe9d, entries=200, sequenceid=80, filesize=38.6 K 2024-12-10T12:34:04,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=52 2024-12-10T12:34:04,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:04,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:04,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:04,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=52}] handler.RSProcedureHandler(58): pid=52 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=52 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/7d440809a4884eddb3563f4d2f10b594 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7d440809a4884eddb3563f4d2f10b594 2024-12-10T12:34:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=52 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:04,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7d440809a4884eddb3563f4d2f10b594, entries=150, sequenceid=80, filesize=11.7 K 2024-12-10T12:34:04,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/c5fb5b4ff9214ee5a9e115fbb306dca3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/c5fb5b4ff9214ee5a9e115fbb306dca3 2024-12-10T12:34:04,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/c5fb5b4ff9214ee5a9e115fbb306dca3, entries=150, sequenceid=80, filesize=11.7 K 2024-12-10T12:34:04,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 946ms, sequenceid=80, compaction requested=false 2024-12-10T12:34:04,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:04,234 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:04,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=52 2024-12-10T12:34:04,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:04,234 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-10T12:34:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:04,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100412ed0802ae40b4b0e1c1920dcdca8c_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834043212/Put/seqid=0 2024-12-10T12:34:04,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741979_1155 (size=12154) 2024-12-10T12:34:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:04,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:04,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834104349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834104351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834104351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834104352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834104352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-10T12:34:04,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834104453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834104456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834104456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834104457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834104456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834104657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:04,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834104660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834104661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834104661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834104661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,665 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100412ed0802ae40b4b0e1c1920dcdca8c_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100412ed0802ae40b4b0e1c1920dcdca8c_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:04,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/2ee481253bd449b8878dfdb5711b69f0, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:04,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/2ee481253bd449b8878dfdb5711b69f0 is 175, key is test_row_0/A:col10/1733834043212/Put/seqid=0 2024-12-10T12:34:04,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741980_1156 (size=30955) 2024-12-10T12:34:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-10T12:34:04,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834104960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834104963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834104964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834104964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:04,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834104965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:05,086 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/2ee481253bd449b8878dfdb5711b69f0 2024-12-10T12:34:05,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/d8ada2b2dd114f03894cdc7ce15d38e6 is 50, key is test_row_0/B:col10/1733834043212/Put/seqid=0 2024-12-10T12:34:05,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741981_1157 (size=12001) 2024-12-10T12:34:05,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834105465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:05,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834105465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:05,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:05,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834105467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834105467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:05,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834105474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:05,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/d8ada2b2dd114f03894cdc7ce15d38e6 2024-12-10T12:34:05,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/45cf39fc38d045958d7ef46a36f08830 is 50, key is test_row_0/C:col10/1733834043212/Put/seqid=0 2024-12-10T12:34:05,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741982_1158 (size=12001) 2024-12-10T12:34:05,528 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/45cf39fc38d045958d7ef46a36f08830 2024-12-10T12:34:05,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/2ee481253bd449b8878dfdb5711b69f0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2ee481253bd449b8878dfdb5711b69f0 2024-12-10T12:34:05,550 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2ee481253bd449b8878dfdb5711b69f0, entries=150, sequenceid=95, filesize=30.2 K 2024-12-10T12:34:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/d8ada2b2dd114f03894cdc7ce15d38e6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d8ada2b2dd114f03894cdc7ce15d38e6 2024-12-10T12:34:05,556 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d8ada2b2dd114f03894cdc7ce15d38e6, entries=150, sequenceid=95, filesize=11.7 K 2024-12-10T12:34:05,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/45cf39fc38d045958d7ef46a36f08830 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/45cf39fc38d045958d7ef46a36f08830 2024-12-10T12:34:05,563 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/45cf39fc38d045958d7ef46a36f08830, entries=150, sequenceid=95, filesize=11.7 K 2024-12-10T12:34:05,564 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1330ms, sequenceid=95, compaction requested=true 2024-12-10T12:34:05,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:05,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:05,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=52 2024-12-10T12:34:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=52 2024-12-10T12:34:05,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-12-10T12:34:05,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7910 sec 2024-12-10T12:34:05,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees in 1.7950 sec 2024-12-10T12:34:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-10T12:34:05,878 INFO [Thread-667 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 51 completed 2024-12-10T12:34:05,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=53, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees 2024-12-10T12:34:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-10T12:34:05,881 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=53, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:05,881 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=53, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:05,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-10T12:34:06,033 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:06,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=54 2024-12-10T12:34:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:06,033 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-10T12:34:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:06,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210efa750cbb7f9421ab1c479b14ad5d234_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834044349/Put/seqid=0 2024-12-10T12:34:06,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741983_1159 (size=12154) 2024-12-10T12:34:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-10T12:34:06,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:06,450 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210efa750cbb7f9421ab1c479b14ad5d234_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210efa750cbb7f9421ab1c479b14ad5d234_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:06,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/fb16fa8141db47d48633b0befed1fa0e, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:06,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/fb16fa8141db47d48633b0befed1fa0e is 175, key is test_row_0/A:col10/1733834044349/Put/seqid=0 2024-12-10T12:34:06,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741984_1160 (size=30955) 2024-12-10T12:34:06,459 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/fb16fa8141db47d48633b0befed1fa0e 2024-12-10T12:34:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/664d4694272c41718f7a8549bda502da is 50, key is test_row_0/B:col10/1733834044349/Put/seqid=0 2024-12-10T12:34:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:06,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:06,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741985_1161 (size=12001) 2024-12-10T12:34:06,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/664d4694272c41718f7a8549bda502da 2024-12-10T12:34:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-10T12:34:06,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834106480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834106480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834106481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/e89d0b6fc99b41e6b5da65a55574245d is 50, key is test_row_0/C:col10/1733834044349/Put/seqid=0 2024-12-10T12:34:06,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834106483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834106486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741986_1162 (size=12001) 2024-12-10T12:34:06,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/e89d0b6fc99b41e6b5da65a55574245d 2024-12-10T12:34:06,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/fb16fa8141db47d48633b0befed1fa0e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/fb16fa8141db47d48633b0befed1fa0e 2024-12-10T12:34:06,514 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/fb16fa8141db47d48633b0befed1fa0e, entries=150, sequenceid=119, filesize=30.2 K 2024-12-10T12:34:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/664d4694272c41718f7a8549bda502da as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/664d4694272c41718f7a8549bda502da 2024-12-10T12:34:06,523 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/664d4694272c41718f7a8549bda502da, entries=150, sequenceid=119, filesize=11.7 K 2024-12-10T12:34:06,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/e89d0b6fc99b41e6b5da65a55574245d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/e89d0b6fc99b41e6b5da65a55574245d 2024-12-10T12:34:06,531 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/e89d0b6fc99b41e6b5da65a55574245d, entries=150, sequenceid=119, filesize=11.7 K 2024-12-10T12:34:06,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 499ms, sequenceid=119, compaction requested=true 2024-12-10T12:34:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=54 2024-12-10T12:34:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=54 2024-12-10T12:34:06,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=54, resume processing ppid=53 2024-12-10T12:34:06,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, ppid=53, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 652 msec 2024-12-10T12:34:06,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees in 655 msec 2024-12-10T12:34:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:06,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-10T12:34:06,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:06,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834106604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834106605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834106606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834106607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bce1d780aed741d18f1f95c67b354f7a_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834046481/Put/seqid=0 2024-12-10T12:34:06,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741987_1163 (size=14794) 2024-12-10T12:34:06,644 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:06,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bce1d780aed741d18f1f95c67b354f7a_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bce1d780aed741d18f1f95c67b354f7a_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:06,652 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f13ee1f8a78842c3b090c592f75851d1, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:06,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f13ee1f8a78842c3b090c592f75851d1 is 175, key is test_row_0/A:col10/1733834046481/Put/seqid=0 2024-12-10T12:34:06,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741988_1164 (size=39749) 2024-12-10T12:34:06,675 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f13ee1f8a78842c3b090c592f75851d1 2024-12-10T12:34:06,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/862014d977a14157a69b8c209662ee60 is 50, key is test_row_0/B:col10/1733834046481/Put/seqid=0 2024-12-10T12:34:06,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741989_1165 (size=12151) 2024-12-10T12:34:06,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/862014d977a14157a69b8c209662ee60 2024-12-10T12:34:06,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/74e74b31a6804134befe0e20d5f23ce2 is 50, key is test_row_0/C:col10/1733834046481/Put/seqid=0 2024-12-10T12:34:06,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834106708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834106708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834106710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834106710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741990_1166 (size=12151) 2024-12-10T12:34:06,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/74e74b31a6804134befe0e20d5f23ce2 2024-12-10T12:34:06,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f13ee1f8a78842c3b090c592f75851d1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f13ee1f8a78842c3b090c592f75851d1 2024-12-10T12:34:06,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f13ee1f8a78842c3b090c592f75851d1, entries=200, sequenceid=134, filesize=38.8 K 2024-12-10T12:34:06,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/862014d977a14157a69b8c209662ee60 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/862014d977a14157a69b8c209662ee60 2024-12-10T12:34:06,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/862014d977a14157a69b8c209662ee60, entries=150, sequenceid=134, filesize=11.9 K 2024-12-10T12:34:06,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/74e74b31a6804134befe0e20d5f23ce2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/74e74b31a6804134befe0e20d5f23ce2 2024-12-10T12:34:06,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/74e74b31a6804134befe0e20d5f23ce2, entries=150, sequenceid=134, filesize=11.9 K 2024-12-10T12:34:06,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 167ms, sequenceid=134, compaction requested=true 2024-12-10T12:34:06,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:06,757 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T12:34:06,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:06,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:06,758 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T12:34:06,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:06,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:06,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:06,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:06,759 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172266 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T12:34:06,760 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/A is initiating minor compaction (all files) 2024-12-10T12:34:06,760 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/A in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:06,760 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b0eb84cb7301490b859f98c4d4bf9fae, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c5a12f966b384043ad3c33fa954efe9d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2ee481253bd449b8878dfdb5711b69f0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/fb16fa8141db47d48633b0befed1fa0e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f13ee1f8a78842c3b090c592f75851d1] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=168.2 K 2024-12-10T12:34:06,760 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:06,760 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b0eb84cb7301490b859f98c4d4bf9fae, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c5a12f966b384043ad3c33fa954efe9d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2ee481253bd449b8878dfdb5711b69f0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/fb16fa8141db47d48633b0befed1fa0e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f13ee1f8a78842c3b090c592f75851d1] 2024-12-10T12:34:06,760 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting b0eb84cb7301490b859f98c4d4bf9fae, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733834042988 2024-12-10T12:34:06,761 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting c5a12f966b384043ad3c33fa954efe9d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733834043041 2024-12-10T12:34:06,761 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60258 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T12:34:06,761 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2ee481253bd449b8878dfdb5711b69f0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733834043212 2024-12-10T12:34:06,761 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/B is initiating minor compaction (all files) 2024-12-10T12:34:06,761 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/B in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:06,761 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d19a5feed7b5401895bdde4e62d2ed15, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7d440809a4884eddb3563f4d2f10b594, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d8ada2b2dd114f03894cdc7ce15d38e6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/664d4694272c41718f7a8549bda502da, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/862014d977a14157a69b8c209662ee60] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=58.8 K 2024-12-10T12:34:06,762 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting fb16fa8141db47d48633b0befed1fa0e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733834044349 2024-12-10T12:34:06,762 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting d19a5feed7b5401895bdde4e62d2ed15, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733834042988 2024-12-10T12:34:06,762 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f13ee1f8a78842c3b090c592f75851d1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733834046481 2024-12-10T12:34:06,763 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 7d440809a4884eddb3563f4d2f10b594, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733834043041 2024-12-10T12:34:06,763 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting d8ada2b2dd114f03894cdc7ce15d38e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733834043212 2024-12-10T12:34:06,764 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 664d4694272c41718f7a8549bda502da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733834044349 2024-12-10T12:34:06,765 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 862014d977a14157a69b8c209662ee60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733834046481 2024-12-10T12:34:06,775 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:06,782 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#B#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:06,782 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/b5725f4ff3064dc488ea2eb2a59c4eb8 is 50, key is test_row_0/B:col10/1733834046481/Put/seqid=0 2024-12-10T12:34:06,788 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412107b8c51ded4424ea2bf915298112e2743_d32afc9a2cd1759fc0fd2b0a98d70e22 store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:06,792 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412107b8c51ded4424ea2bf915298112e2743_d32afc9a2cd1759fc0fd2b0a98d70e22, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:06,793 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107b8c51ded4424ea2bf915298112e2743_d32afc9a2cd1759fc0fd2b0a98d70e22 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:06,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741991_1167 (size=12425) 2024-12-10T12:34:06,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741992_1168 (size=4469) 2024-12-10T12:34:06,807 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#A#compaction#141 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:06,808 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/99b5bbf893d5463f979e686870161cbb is 175, key is test_row_0/A:col10/1733834046481/Put/seqid=0 2024-12-10T12:34:06,812 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/b5725f4ff3064dc488ea2eb2a59c4eb8 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b5725f4ff3064dc488ea2eb2a59c4eb8 2024-12-10T12:34:06,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741993_1169 (size=31379) 2024-12-10T12:34:06,824 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 5 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/B of d32afc9a2cd1759fc0fd2b0a98d70e22 into b5725f4ff3064dc488ea2eb2a59c4eb8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:06,824 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:06,824 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/B, priority=11, startTime=1733834046758; duration=0sec 2024-12-10T12:34:06,824 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:06,824 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:B 2024-12-10T12:34:06,824 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T12:34:06,826 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/99b5bbf893d5463f979e686870161cbb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/99b5bbf893d5463f979e686870161cbb 2024-12-10T12:34:06,827 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60258 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T12:34:06,827 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/C is initiating minor compaction (all files) 2024-12-10T12:34:06,827 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/C in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:06,827 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/be7c1dd5d80a40c2b767238d19bf2fcf, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/c5fb5b4ff9214ee5a9e115fbb306dca3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/45cf39fc38d045958d7ef46a36f08830, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/e89d0b6fc99b41e6b5da65a55574245d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/74e74b31a6804134befe0e20d5f23ce2] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=58.8 K 2024-12-10T12:34:06,828 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting be7c1dd5d80a40c2b767238d19bf2fcf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733834042988 2024-12-10T12:34:06,828 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting c5fb5b4ff9214ee5a9e115fbb306dca3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733834043041 2024-12-10T12:34:06,829 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 45cf39fc38d045958d7ef46a36f08830, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733834043212 2024-12-10T12:34:06,830 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting e89d0b6fc99b41e6b5da65a55574245d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733834044349 2024-12-10T12:34:06,831 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 74e74b31a6804134befe0e20d5f23ce2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733834046481 2024-12-10T12:34:06,834 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 5 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/A of d32afc9a2cd1759fc0fd2b0a98d70e22 into 99b5bbf893d5463f979e686870161cbb(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:06,834 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:06,834 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/A, priority=11, startTime=1733834046757; duration=0sec 2024-12-10T12:34:06,834 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:06,834 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:A 2024-12-10T12:34:06,847 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#C#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:06,847 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/996b3d8a499744928fbe156a10a7c26f is 50, key is test_row_0/C:col10/1733834046481/Put/seqid=0 2024-12-10T12:34:06,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741994_1170 (size=12425) 2024-12-10T12:34:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:06,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-10T12:34:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:06,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cc2fad51ab9e488c96a99bf2fb682a22_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834046915/Put/seqid=0 2024-12-10T12:34:06,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834106933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834106937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834106938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834106938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:06,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741995_1171 (size=17284) 2024-12-10T12:34:06,949 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:06,955 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cc2fad51ab9e488c96a99bf2fb682a22_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cc2fad51ab9e488c96a99bf2fb682a22_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:06,956 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/dba5b60a005d4e2f87b3d9f513a8d58e, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:06,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/dba5b60a005d4e2f87b3d9f513a8d58e is 175, key is test_row_0/A:col10/1733834046915/Put/seqid=0 2024-12-10T12:34:06,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741996_1172 (size=48389) 2024-12-10T12:34:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-10T12:34:06,984 INFO [Thread-667 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 53 completed 2024-12-10T12:34:06,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=55, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees 2024-12-10T12:34:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-10T12:34:06,990 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=55, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:06,991 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=55, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:06,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:07,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834107042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834107042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834107042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834107042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-10T12:34:07,143 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:07,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-12-10T12:34:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834107247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834107248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834107248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834107248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,276 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/996b3d8a499744928fbe156a10a7c26f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/996b3d8a499744928fbe156a10a7c26f 2024-12-10T12:34:07,282 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 5 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/C of d32afc9a2cd1759fc0fd2b0a98d70e22 into 996b3d8a499744928fbe156a10a7c26f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:07,283 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:07,283 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/C, priority=11, startTime=1733834046759; duration=0sec 2024-12-10T12:34:07,284 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:07,284 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:C 2024-12-10T12:34:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-10T12:34:07,297 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:07,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-12-10T12:34:07,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:07,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,370 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/dba5b60a005d4e2f87b3d9f513a8d58e 2024-12-10T12:34:07,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/5f6ea613e1384ce6b700de21aed177a2 is 50, key is test_row_0/B:col10/1733834046915/Put/seqid=0 2024-12-10T12:34:07,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741997_1173 (size=12151) 2024-12-10T12:34:07,450 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:07,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-12-10T12:34:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834107549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834107551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834107551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834107554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-10T12:34:07,611 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:07,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-12-10T12:34:07,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:07,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,763 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:07,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-12-10T12:34:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:07,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/5f6ea613e1384ce6b700de21aed177a2 2024-12-10T12:34:07,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/bc5503bb8d7e4738ad6a9f4449e5a033 is 50, key is test_row_0/C:col10/1733834046915/Put/seqid=0 2024-12-10T12:34:07,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741998_1174 (size=12151) 2024-12-10T12:34:07,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/bc5503bb8d7e4738ad6a9f4449e5a033 2024-12-10T12:34:07,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/dba5b60a005d4e2f87b3d9f513a8d58e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/dba5b60a005d4e2f87b3d9f513a8d58e 2024-12-10T12:34:07,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/dba5b60a005d4e2f87b3d9f513a8d58e, entries=250, sequenceid=161, filesize=47.3 K 2024-12-10T12:34:07,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/5f6ea613e1384ce6b700de21aed177a2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/5f6ea613e1384ce6b700de21aed177a2 2024-12-10T12:34:07,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/5f6ea613e1384ce6b700de21aed177a2, entries=150, sequenceid=161, filesize=11.9 K 2024-12-10T12:34:07,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/bc5503bb8d7e4738ad6a9f4449e5a033 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc5503bb8d7e4738ad6a9f4449e5a033 2024-12-10T12:34:07,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc5503bb8d7e4738ad6a9f4449e5a033, entries=150, sequenceid=161, filesize=11.9 K 2024-12-10T12:34:07,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 916ms, sequenceid=161, compaction requested=false 2024-12-10T12:34:07,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:07,916 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:07,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-12-10T12:34:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:07,917 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:34:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:07,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:07,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121016cc5059c3d24f5292cc3132b1b2611b_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834046924/Put/seqid=0 2024-12-10T12:34:07,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741999_1175 (size=12304) 2024-12-10T12:34:08,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:08,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:08,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834108069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834108070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834108071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834108072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-10T12:34:08,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834108173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834108173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834108174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834108174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:08,342 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121016cc5059c3d24f5292cc3132b1b2611b_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121016cc5059c3d24f5292cc3132b1b2611b_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:08,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/0e9f0cee67ff45cea2326c7205bbe0b2, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:08,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/0e9f0cee67ff45cea2326c7205bbe0b2 is 175, key is test_row_0/A:col10/1733834046924/Put/seqid=0 2024-12-10T12:34:08,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742000_1176 (size=31105) 2024-12-10T12:34:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834108375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834108375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834108375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834108376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834108498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,499 DEBUG [Thread-665 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4147 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:34:08,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834108677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834108679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834108680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834108681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:08,749 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/0e9f0cee67ff45cea2326c7205bbe0b2 2024-12-10T12:34:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/bd3bf922ed324767aadbf773060b4385 is 50, key is test_row_0/B:col10/1733834046924/Put/seqid=0 2024-12-10T12:34:08,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742001_1177 (size=12151) 2024-12-10T12:34:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-10T12:34:09,159 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/bd3bf922ed324767aadbf773060b4385 2024-12-10T12:34:09,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/5317a82b323047f9b5ee0458fb0e02fe is 50, key is test_row_0/C:col10/1733834046924/Put/seqid=0 2024-12-10T12:34:09,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742002_1178 (size=12151) 2024-12-10T12:34:09,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834109184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:09,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834109187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:09,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834109187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:09,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834109190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:09,572 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/5317a82b323047f9b5ee0458fb0e02fe 2024-12-10T12:34:09,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/0e9f0cee67ff45cea2326c7205bbe0b2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/0e9f0cee67ff45cea2326c7205bbe0b2 2024-12-10T12:34:09,581 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/0e9f0cee67ff45cea2326c7205bbe0b2, entries=150, sequenceid=175, filesize=30.4 K 2024-12-10T12:34:09,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/bd3bf922ed324767aadbf773060b4385 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/bd3bf922ed324767aadbf773060b4385 2024-12-10T12:34:09,586 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/bd3bf922ed324767aadbf773060b4385, entries=150, sequenceid=175, filesize=11.9 K 2024-12-10T12:34:09,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/5317a82b323047f9b5ee0458fb0e02fe as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5317a82b323047f9b5ee0458fb0e02fe 2024-12-10T12:34:09,592 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5317a82b323047f9b5ee0458fb0e02fe, entries=150, sequenceid=175, filesize=11.9 K 2024-12-10T12:34:09,592 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1675ms, sequenceid=175, compaction requested=true 2024-12-10T12:34:09,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:09,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:09,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=56 2024-12-10T12:34:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=56 2024-12-10T12:34:09,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-12-10T12:34:09,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6030 sec 2024-12-10T12:34:09,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees in 2.6070 sec 2024-12-10T12:34:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:10,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-10T12:34:10,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:10,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:10,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:10,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:10,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:10,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:10,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210195db30f3b8c4c788bbcb59354a82925_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834048068/Put/seqid=0 2024-12-10T12:34:10,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834110202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834110202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834110203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742003_1179 (size=14794) 2024-12-10T12:34:10,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834110211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834110305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834110305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834110306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834110314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834110507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834110508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834110508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834110517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,606 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:10,610 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210195db30f3b8c4c788bbcb59354a82925_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210195db30f3b8c4c788bbcb59354a82925_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:10,611 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f61a243cc6af4118a26d3939da26153a, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:10,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f61a243cc6af4118a26d3939da26153a is 175, key is test_row_0/A:col10/1733834048068/Put/seqid=0 2024-12-10T12:34:10,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742004_1180 (size=39749) 2024-12-10T12:34:10,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834110810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834110811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834110811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:10,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834110818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:11,016 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f61a243cc6af4118a26d3939da26153a 2024-12-10T12:34:11,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/20d31a2826c84689aab19eb6223a8e4b is 50, key is test_row_0/B:col10/1733834048068/Put/seqid=0 2024-12-10T12:34:11,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742005_1181 (size=12151) 2024-12-10T12:34:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-10T12:34:11,094 INFO [Thread-667 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 55 completed 2024-12-10T12:34:11,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees 2024-12-10T12:34:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-10T12:34:11,097 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=57, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:11,097 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=57, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:11,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-10T12:34:11,248 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:11,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=58 2024-12-10T12:34:11,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:11,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] handler.RSProcedureHandler(58): pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=58 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834111311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:11,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:11,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834111314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834111314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:11,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834111320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-10T12:34:11,401 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:11,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=58 2024-12-10T12:34:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] handler.RSProcedureHandler(58): pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=58 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/20d31a2826c84689aab19eb6223a8e4b 2024-12-10T12:34:11,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/a80f562fb84140daa0497c914ae45989 is 50, key is test_row_0/C:col10/1733834048068/Put/seqid=0 2024-12-10T12:34:11,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742006_1182 (size=12151) 2024-12-10T12:34:11,553 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:11,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=58 2024-12-10T12:34:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] handler.RSProcedureHandler(58): pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=58 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-10T12:34:11,706 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:11,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=58 2024-12-10T12:34:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] handler.RSProcedureHandler(58): pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=58 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/a80f562fb84140daa0497c914ae45989 2024-12-10T12:34:11,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/f61a243cc6af4118a26d3939da26153a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f61a243cc6af4118a26d3939da26153a 2024-12-10T12:34:11,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f61a243cc6af4118a26d3939da26153a, entries=200, sequenceid=199, filesize=38.8 K 2024-12-10T12:34:11,860 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:11,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=58 2024-12-10T12:34:11,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/20d31a2826c84689aab19eb6223a8e4b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/20d31a2826c84689aab19eb6223a8e4b 2024-12-10T12:34:11,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:11,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] handler.RSProcedureHandler(58): pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=58 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=58 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:11,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/20d31a2826c84689aab19eb6223a8e4b, entries=150, sequenceid=199, filesize=11.9 K 2024-12-10T12:34:11,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/a80f562fb84140daa0497c914ae45989 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/a80f562fb84140daa0497c914ae45989 2024-12-10T12:34:11,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/a80f562fb84140daa0497c914ae45989, entries=150, sequenceid=199, filesize=11.9 K 2024-12-10T12:34:11,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1680ms, sequenceid=199, compaction requested=true 2024-12-10T12:34:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:11,871 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:11,871 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:11,872 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150622 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:11,872 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/A is initiating minor compaction (all files) 2024-12-10T12:34:11,873 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/A in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,873 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/99b5bbf893d5463f979e686870161cbb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/dba5b60a005d4e2f87b3d9f513a8d58e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/0e9f0cee67ff45cea2326c7205bbe0b2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f61a243cc6af4118a26d3939da26153a] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=147.1 K 2024-12-10T12:34:11,873 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:11,873 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,873 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/B is initiating minor compaction (all files) 2024-12-10T12:34:11,873 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/99b5bbf893d5463f979e686870161cbb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/dba5b60a005d4e2f87b3d9f513a8d58e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/0e9f0cee67ff45cea2326c7205bbe0b2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f61a243cc6af4118a26d3939da26153a] 2024-12-10T12:34:11,873 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/B in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,873 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b5725f4ff3064dc488ea2eb2a59c4eb8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/5f6ea613e1384ce6b700de21aed177a2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/bd3bf922ed324767aadbf773060b4385, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/20d31a2826c84689aab19eb6223a8e4b] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=47.7 K 2024-12-10T12:34:11,873 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 99b5bbf893d5463f979e686870161cbb, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733834046481 2024-12-10T12:34:11,873 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting b5725f4ff3064dc488ea2eb2a59c4eb8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733834046481 2024-12-10T12:34:11,874 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting dba5b60a005d4e2f87b3d9f513a8d58e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733834046605 2024-12-10T12:34:11,874 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5f6ea613e1384ce6b700de21aed177a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733834046605 2024-12-10T12:34:11,874 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0e9f0cee67ff45cea2326c7205bbe0b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733834046924 2024-12-10T12:34:11,874 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting bd3bf922ed324767aadbf773060b4385, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733834046924 2024-12-10T12:34:11,874 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f61a243cc6af4118a26d3939da26153a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834048068 2024-12-10T12:34:11,875 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 20d31a2826c84689aab19eb6223a8e4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834048068 2024-12-10T12:34:11,881 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:11,884 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412108ed10708498440e7a986499c9a6f317a_d32afc9a2cd1759fc0fd2b0a98d70e22 store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:11,885 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:11,885 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412108ed10708498440e7a986499c9a6f317a_d32afc9a2cd1759fc0fd2b0a98d70e22, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:11,885 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/52f1377349714408936e841d23459c5c is 50, key is test_row_0/B:col10/1733834048068/Put/seqid=0 2024-12-10T12:34:11,886 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108ed10708498440e7a986499c9a6f317a_d32afc9a2cd1759fc0fd2b0a98d70e22 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:11,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742008_1184 (size=4469) 2024-12-10T12:34:11,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742007_1183 (size=12561) 2024-12-10T12:34:11,891 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#A#compaction#153 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:11,892 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/407d26c1b12c4a478faa2631217cfc44 is 175, key is test_row_0/A:col10/1733834048068/Put/seqid=0 2024-12-10T12:34:11,897 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/52f1377349714408936e841d23459c5c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/52f1377349714408936e841d23459c5c 2024-12-10T12:34:11,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742009_1185 (size=31515) 2024-12-10T12:34:11,902 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/B of d32afc9a2cd1759fc0fd2b0a98d70e22 into 52f1377349714408936e841d23459c5c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:11,902 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:11,902 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/B, priority=12, startTime=1733834051871; duration=0sec 2024-12-10T12:34:11,902 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:11,902 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:B 2024-12-10T12:34:11,902 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:11,904 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:11,904 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/C is initiating minor compaction (all files) 2024-12-10T12:34:11,904 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/C in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:11,904 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/996b3d8a499744928fbe156a10a7c26f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc5503bb8d7e4738ad6a9f4449e5a033, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5317a82b323047f9b5ee0458fb0e02fe, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/a80f562fb84140daa0497c914ae45989] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=47.7 K 2024-12-10T12:34:11,904 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 996b3d8a499744928fbe156a10a7c26f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733834046481 2024-12-10T12:34:11,905 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting bc5503bb8d7e4738ad6a9f4449e5a033, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733834046605 2024-12-10T12:34:11,905 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5317a82b323047f9b5ee0458fb0e02fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733834046924 2024-12-10T12:34:11,906 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting a80f562fb84140daa0497c914ae45989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834048068 2024-12-10T12:34:11,914 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:11,915 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/8bfd65e86b754f77b627a74f78d9d1b6 is 50, key is test_row_0/C:col10/1733834048068/Put/seqid=0 2024-12-10T12:34:11,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742010_1186 (size=12561) 2024-12-10T12:34:12,013 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:12,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=58 2024-12-10T12:34:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:12,014 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-10T12:34:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:12,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105dac59b7940a4b42b84ec3fbe5c61d7a_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834050200/Put/seqid=0 2024-12-10T12:34:12,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742011_1187 (size=12304) 2024-12-10T12:34:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-10T12:34:12,302 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/407d26c1b12c4a478faa2631217cfc44 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/407d26c1b12c4a478faa2631217cfc44 2024-12-10T12:34:12,307 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/A of d32afc9a2cd1759fc0fd2b0a98d70e22 into 407d26c1b12c4a478faa2631217cfc44(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:12,307 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:12,307 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/A, priority=12, startTime=1733834051871; duration=0sec 2024-12-10T12:34:12,307 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:12,307 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:A 2024-12-10T12:34:12,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:12,324 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/8bfd65e86b754f77b627a74f78d9d1b6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/8bfd65e86b754f77b627a74f78d9d1b6 2024-12-10T12:34:12,329 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/C of d32afc9a2cd1759fc0fd2b0a98d70e22 into 8bfd65e86b754f77b627a74f78d9d1b6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:12,329 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:12,329 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/C, priority=12, startTime=1733834051871; duration=0sec 2024-12-10T12:34:12,329 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:12,329 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:C 2024-12-10T12:34:12,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834112336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834112336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834112336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834112337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:12,437 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105dac59b7940a4b42b84ec3fbe5c61d7a_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105dac59b7940a4b42b84ec3fbe5c61d7a_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:12,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/6e6c0ed3037e430bb02a508ecc1f582d, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834112438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/6e6c0ed3037e430bb02a508ecc1f582d is 175, key is test_row_0/A:col10/1733834050200/Put/seqid=0 2024-12-10T12:34:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834112439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834112440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834112440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742012_1188 (size=31105) 2024-12-10T12:34:12,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834112505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,507 DEBUG [Thread-665 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8154 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:34:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834112640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834112641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834112642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834112642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,844 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/6e6c0ed3037e430bb02a508ecc1f582d 2024-12-10T12:34:12,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/b139d869e093470a92c643703403cd63 is 50, key is test_row_0/B:col10/1733834050200/Put/seqid=0 2024-12-10T12:34:12,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742013_1189 (size=12151) 2024-12-10T12:34:12,854 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/b139d869e093470a92c643703403cd63 2024-12-10T12:34:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/66f6d4b27e694830af2b6ee145b02dc6 is 50, key is test_row_0/C:col10/1733834050200/Put/seqid=0 2024-12-10T12:34:12,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742014_1190 (size=12151) 2024-12-10T12:34:12,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834112944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834112945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834112946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:12,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834112946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-10T12:34:13,267 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/66f6d4b27e694830af2b6ee145b02dc6 2024-12-10T12:34:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/6e6c0ed3037e430bb02a508ecc1f582d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/6e6c0ed3037e430bb02a508ecc1f582d 2024-12-10T12:34:13,275 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/6e6c0ed3037e430bb02a508ecc1f582d, entries=150, sequenceid=212, filesize=30.4 K 2024-12-10T12:34:13,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/b139d869e093470a92c643703403cd63 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b139d869e093470a92c643703403cd63 2024-12-10T12:34:13,280 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b139d869e093470a92c643703403cd63, entries=150, sequenceid=212, filesize=11.9 K 2024-12-10T12:34:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/66f6d4b27e694830af2b6ee145b02dc6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/66f6d4b27e694830af2b6ee145b02dc6 2024-12-10T12:34:13,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/66f6d4b27e694830af2b6ee145b02dc6, entries=150, sequenceid=212, filesize=11.9 K 2024-12-10T12:34:13,286 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1272ms, sequenceid=212, compaction requested=false 2024-12-10T12:34:13,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:13,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:13,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58 2024-12-10T12:34:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=58 2024-12-10T12:34:13,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-12-10T12:34:13,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1900 sec 2024-12-10T12:34:13,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees in 2.1930 sec 2024-12-10T12:34:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:13,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-10T12:34:13,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:13,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:13,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:13,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:13,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:13,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:13,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834113452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834113453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d7f69a24670d4ef6a3f60e5bbecf241a_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834052335/Put/seqid=0 2024-12-10T12:34:13,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834113455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834113457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742015_1191 (size=14794) 2024-12-10T12:34:13,512 INFO [master/fad905551524:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-10T12:34:13,512 INFO [master/fad905551524:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-10T12:34:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834113556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834113556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834113558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834113558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834113757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834113758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834113761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834113761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:13,861 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:13,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d7f69a24670d4ef6a3f60e5bbecf241a_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d7f69a24670d4ef6a3f60e5bbecf241a_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:13,865 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/68d20620347d48a299c3147d17cc9c97, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:13,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/68d20620347d48a299c3147d17cc9c97 is 175, key is test_row_0/A:col10/1733834052335/Put/seqid=0 2024-12-10T12:34:13,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742016_1192 (size=39749) 2024-12-10T12:34:14,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834114059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:14,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834114061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:14,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834114063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:14,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834114064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:14,271 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/68d20620347d48a299c3147d17cc9c97 2024-12-10T12:34:14,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/ea043f003a5d4869882f946516b98261 is 50, key is test_row_0/B:col10/1733834052335/Put/seqid=0 2024-12-10T12:34:14,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742017_1193 (size=12151) 2024-12-10T12:34:14,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834114562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:14,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834114566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:14,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834114566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:14,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834114568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:14,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/ea043f003a5d4869882f946516b98261 2024-12-10T12:34:14,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/3c5993b0bf0043d0b165b607d15df152 is 50, key is test_row_0/C:col10/1733834052335/Put/seqid=0 2024-12-10T12:34:14,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742018_1194 (size=12151) 2024-12-10T12:34:15,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/3c5993b0bf0043d0b165b607d15df152 2024-12-10T12:34:15,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/68d20620347d48a299c3147d17cc9c97 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/68d20620347d48a299c3147d17cc9c97 2024-12-10T12:34:15,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/68d20620347d48a299c3147d17cc9c97, entries=200, sequenceid=241, filesize=38.8 K 2024-12-10T12:34:15,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/ea043f003a5d4869882f946516b98261 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/ea043f003a5d4869882f946516b98261 2024-12-10T12:34:15,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/ea043f003a5d4869882f946516b98261, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T12:34:15,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/3c5993b0bf0043d0b165b607d15df152 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/3c5993b0bf0043d0b165b607d15df152 2024-12-10T12:34:15,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/3c5993b0bf0043d0b165b607d15df152, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T12:34:15,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1667ms, sequenceid=241, compaction requested=true 2024-12-10T12:34:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:15,117 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:15,117 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:15,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:15,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:15,118 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:15,118 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/A is initiating minor compaction (all files) 2024-12-10T12:34:15,118 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/A in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:15,118 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/407d26c1b12c4a478faa2631217cfc44, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/6e6c0ed3037e430bb02a508ecc1f582d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/68d20620347d48a299c3147d17cc9c97] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=100.0 K 2024-12-10T12:34:15,118 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:15,118 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/407d26c1b12c4a478faa2631217cfc44, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/6e6c0ed3037e430bb02a508ecc1f582d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/68d20620347d48a299c3147d17cc9c97] 2024-12-10T12:34:15,119 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:15,119 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/B is initiating minor compaction (all files) 2024-12-10T12:34:15,119 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/B in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:15,119 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/52f1377349714408936e841d23459c5c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b139d869e093470a92c643703403cd63, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/ea043f003a5d4869882f946516b98261] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=36.0 K 2024-12-10T12:34:15,119 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 407d26c1b12c4a478faa2631217cfc44, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834048068 2024-12-10T12:34:15,119 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 52f1377349714408936e841d23459c5c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834048068 2024-12-10T12:34:15,119 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6e6c0ed3037e430bb02a508ecc1f582d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733834050200 2024-12-10T12:34:15,120 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting b139d869e093470a92c643703403cd63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733834050200 2024-12-10T12:34:15,120 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 68d20620347d48a299c3147d17cc9c97, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733834052335 2024-12-10T12:34:15,120 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ea043f003a5d4869882f946516b98261, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733834052335 2024-12-10T12:34:15,126 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:15,128 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121020a8bb63213045b7941808db276cf7d3_d32afc9a2cd1759fc0fd2b0a98d70e22 store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:15,130 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121020a8bb63213045b7941808db276cf7d3_d32afc9a2cd1759fc0fd2b0a98d70e22, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:15,130 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121020a8bb63213045b7941808db276cf7d3_d32afc9a2cd1759fc0fd2b0a98d70e22 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:15,131 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:15,132 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/3ca2c616f53a4499b0f1f31d1fbcfb3d is 50, key is test_row_0/B:col10/1733834052335/Put/seqid=0 2024-12-10T12:34:15,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742020_1196 (size=12663) 2024-12-10T12:34:15,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742019_1195 (size=4469) 2024-12-10T12:34:15,165 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/3ca2c616f53a4499b0f1f31d1fbcfb3d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/3ca2c616f53a4499b0f1f31d1fbcfb3d 2024-12-10T12:34:15,168 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#A#compaction#162 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:15,168 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/e58eaddd2a89454286d72e48a6669cd7 is 175, key is test_row_0/A:col10/1733834052335/Put/seqid=0 2024-12-10T12:34:15,178 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/B of d32afc9a2cd1759fc0fd2b0a98d70e22 into 3ca2c616f53a4499b0f1f31d1fbcfb3d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:15,178 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:15,178 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/B, priority=13, startTime=1733834055117; duration=0sec 2024-12-10T12:34:15,178 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:15,178 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:B 2024-12-10T12:34:15,178 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:15,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742021_1197 (size=31617) 2024-12-10T12:34:15,182 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:15,183 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/C is initiating minor compaction (all files) 2024-12-10T12:34:15,183 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/C in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:15,183 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/8bfd65e86b754f77b627a74f78d9d1b6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/66f6d4b27e694830af2b6ee145b02dc6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/3c5993b0bf0043d0b165b607d15df152] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=36.0 K 2024-12-10T12:34:15,184 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 8bfd65e86b754f77b627a74f78d9d1b6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834048068 2024-12-10T12:34:15,185 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 66f6d4b27e694830af2b6ee145b02dc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733834050200 2024-12-10T12:34:15,186 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 3c5993b0bf0043d0b165b607d15df152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733834052335 2024-12-10T12:34:15,189 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/e58eaddd2a89454286d72e48a6669cd7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e58eaddd2a89454286d72e48a6669cd7 2024-12-10T12:34:15,196 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/A of d32afc9a2cd1759fc0fd2b0a98d70e22 into e58eaddd2a89454286d72e48a6669cd7(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:15,196 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:15,196 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/A, priority=13, startTime=1733834055117; duration=0sec 2024-12-10T12:34:15,196 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:15,196 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:A 2024-12-10T12:34:15,197 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#C#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:15,198 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/81e8da9bf06148f48c4cbe9b2f81070b is 50, key is test_row_0/C:col10/1733834052335/Put/seqid=0 2024-12-10T12:34:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-10T12:34:15,201 INFO [Thread-667 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 57 completed 2024-12-10T12:34:15,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=59, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees 2024-12-10T12:34:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-10T12:34:15,205 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=59, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:15,205 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=59, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:15,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:15,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742022_1198 (size=12663) 2024-12-10T12:34:15,215 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/81e8da9bf06148f48c4cbe9b2f81070b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/81e8da9bf06148f48c4cbe9b2f81070b 2024-12-10T12:34:15,223 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/C of d32afc9a2cd1759fc0fd2b0a98d70e22 into 81e8da9bf06148f48c4cbe9b2f81070b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:15,223 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:15,223 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/C, priority=13, startTime=1733834055118; duration=0sec 2024-12-10T12:34:15,224 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:15,224 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:C 2024-12-10T12:34:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-10T12:34:15,359 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:15,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-12-10T12:34:15,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:15,360 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-10T12:34:15,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:15,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:15,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:15,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:15,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:15,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100c761c6b6d0842b09b007e86222408d7_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834053454/Put/seqid=0 2024-12-10T12:34:15,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742023_1199 (size=12304) 2024-12-10T12:34:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-10T12:34:15,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:15,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834115582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834115583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834115584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834115584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834115685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834115686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834115686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834115686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:15,782 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100c761c6b6d0842b09b007e86222408d7_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100c761c6b6d0842b09b007e86222408d7_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:15,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/632eb92b4146422b9ddcf8632aac8d6f, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:15,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/632eb92b4146422b9ddcf8632aac8d6f is 175, key is test_row_0/A:col10/1733834053454/Put/seqid=0 2024-12-10T12:34:15,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742024_1200 (size=31105) 2024-12-10T12:34:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-10T12:34:15,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834115889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834115889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834115889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:15,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834115890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:16,189 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/632eb92b4146422b9ddcf8632aac8d6f 2024-12-10T12:34:16,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834116191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:16,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834116192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:16,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834116194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:16,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834116194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/82bd985d29914a39946389528c639258 is 50, key is test_row_0/B:col10/1733834053454/Put/seqid=0 2024-12-10T12:34:16,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742025_1201 (size=12151) 2024-12-10T12:34:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-10T12:34:16,603 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/82bd985d29914a39946389528c639258 2024-12-10T12:34:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/0bc3ec2b074f4274a515c63f3379138a is 50, key is test_row_0/C:col10/1733834053454/Put/seqid=0 2024-12-10T12:34:16,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742026_1202 (size=12151) 2024-12-10T12:34:16,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834116694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834116695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834116698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834116701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:17,015 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/0bc3ec2b074f4274a515c63f3379138a 2024-12-10T12:34:17,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/632eb92b4146422b9ddcf8632aac8d6f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/632eb92b4146422b9ddcf8632aac8d6f 2024-12-10T12:34:17,023 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/632eb92b4146422b9ddcf8632aac8d6f, entries=150, sequenceid=254, filesize=30.4 K 2024-12-10T12:34:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/82bd985d29914a39946389528c639258 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/82bd985d29914a39946389528c639258 2024-12-10T12:34:17,028 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/82bd985d29914a39946389528c639258, entries=150, sequenceid=254, filesize=11.9 K 2024-12-10T12:34:17,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/0bc3ec2b074f4274a515c63f3379138a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/0bc3ec2b074f4274a515c63f3379138a 2024-12-10T12:34:17,033 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/0bc3ec2b074f4274a515c63f3379138a, entries=150, sequenceid=254, filesize=11.9 K 2024-12-10T12:34:17,035 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(3040): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1675ms, sequenceid=254, compaction requested=false 2024-12-10T12:34:17,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:17,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:17,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-12-10T12:34:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=60 2024-12-10T12:34:17,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=60, resume processing ppid=59 2024-12-10T12:34:17,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, ppid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-12-10T12:34:17,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees in 1.8350 sec 2024-12-10T12:34:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-10T12:34:17,308 INFO [Thread-667 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 59 completed 2024-12-10T12:34:17,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=61, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees 2024-12-10T12:34:17,310 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=61, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-10T12:34:17,310 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=61, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:17,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-10T12:34:17,462 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:17,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-12-10T12:34:17,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:17,462 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-10T12:34:17,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:17,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:17,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:17,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:17,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:17,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:17,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210280c1399a383493093953c607f84a51f_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834055581/Put/seqid=0 2024-12-10T12:34:17,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742027_1203 (size=12454) 2024-12-10T12:34:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:17,481 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210280c1399a383493093953c607f84a51f_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210280c1399a383493093953c607f84a51f_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:17,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/bd8446786c124f62af48dfe191abf728, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/bd8446786c124f62af48dfe191abf728 is 175, key is test_row_0/A:col10/1733834055581/Put/seqid=0 2024-12-10T12:34:17,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742028_1204 (size=31255) 2024-12-10T12:34:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-10T12:34:17,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:17,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834117703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:17,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834117704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:17,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834117704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:17,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834117709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834117806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:17,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834117807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:17,894 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/bd8446786c124f62af48dfe191abf728 2024-12-10T12:34:17,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/2e13a3ad5568478e87553427dedb064e is 50, key is test_row_0/B:col10/1733834055581/Put/seqid=0 2024-12-10T12:34:17,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742029_1205 (size=12301) 2024-12-10T12:34:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-10T12:34:18,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834118007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834118010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:18,305 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/2e13a3ad5568478e87553427dedb064e 2024-12-10T12:34:18,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/6a7a9f1b3e2e4575970745973e62b1f9 is 50, key is test_row_0/C:col10/1733834055581/Put/seqid=0 2024-12-10T12:34:18,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834118312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:18,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834118313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:18,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742030_1206 (size=12301) 2024-12-10T12:34:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-10T12:34:18,716 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/6a7a9f1b3e2e4575970745973e62b1f9 2024-12-10T12:34:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/bd8446786c124f62af48dfe191abf728 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/bd8446786c124f62af48dfe191abf728 2024-12-10T12:34:18,724 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/bd8446786c124f62af48dfe191abf728, entries=150, sequenceid=280, filesize=30.5 K 2024-12-10T12:34:18,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/2e13a3ad5568478e87553427dedb064e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/2e13a3ad5568478e87553427dedb064e 2024-12-10T12:34:18,729 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/2e13a3ad5568478e87553427dedb064e, entries=150, sequenceid=280, filesize=12.0 K 2024-12-10T12:34:18,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/6a7a9f1b3e2e4575970745973e62b1f9 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6a7a9f1b3e2e4575970745973e62b1f9 2024-12-10T12:34:18,734 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6a7a9f1b3e2e4575970745973e62b1f9, entries=150, sequenceid=280, filesize=12.0 K 2024-12-10T12:34:18,735 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1272ms, sequenceid=280, compaction requested=true 2024-12-10T12:34:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-10T12:34:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=62 2024-12-10T12:34:18,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-12-10T12:34:18,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-12-10T12:34:18,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees in 1.4280 sec 2024-12-10T12:34:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:18,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:34:18,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:18,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:18,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:18,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:18,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:18,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:18,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121006ef9651274d47cebdb072617f081a46_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834057701/Put/seqid=0 2024-12-10T12:34:18,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742031_1207 (size=12454) 2024-12-10T12:34:18,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834118858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:18,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834118858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:18,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834118960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:18,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834118960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:19,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834119162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:19,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834119163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:19,230 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:19,233 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121006ef9651274d47cebdb072617f081a46_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121006ef9651274d47cebdb072617f081a46_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:19,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/9dfff2794b97421b905511781e1ab019, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:19,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/9dfff2794b97421b905511781e1ab019 is 175, key is test_row_0/A:col10/1733834057701/Put/seqid=0 2024-12-10T12:34:19,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742032_1208 (size=31255) 2024-12-10T12:34:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-10T12:34:19,414 INFO [Thread-667 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 61 completed 2024-12-10T12:34:19,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=63, table=TestAcidGuarantees 2024-12-10T12:34:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-10T12:34:19,416 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=63, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=63, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:19,416 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=63, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=63, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:19,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:19,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834119464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:19,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834119466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-10T12:34:19,567 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:19,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=64 2024-12-10T12:34:19,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:19,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:19,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:19,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] handler.RSProcedureHandler(58): pid=64 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:19,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=64 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=64 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:19,639 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/9dfff2794b97421b905511781e1ab019 2024-12-10T12:34:19,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/6ba394992e7741f999047f06d2066836 is 50, key is test_row_0/B:col10/1733834057701/Put/seqid=0 2024-12-10T12:34:19,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742033_1209 (size=12301) 2024-12-10T12:34:19,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43338 deadline: 1733834119711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:19,712 DEBUG [Thread-659 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4129 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:34:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-10T12:34:19,720 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:19,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=64 2024-12-10T12:34:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:19,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] handler.RSProcedureHandler(58): pid=64 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=64 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=64 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:19,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43320 deadline: 1733834119727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:19,729 DEBUG [Thread-657 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4146 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:34:19,872 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:19,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=64 2024-12-10T12:34:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:19,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=64}] handler.RSProcedureHandler(58): pid=64 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=64 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=64 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834119968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:19,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834119970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-10T12:34:20,024 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:20,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=64 2024-12-10T12:34:20,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:20,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:20,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:20,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] handler.RSProcedureHandler(58): pid=64 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:20,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=64 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=64 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:20,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/6ba394992e7741f999047f06d2066836 2024-12-10T12:34:20,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/91cdb9760da24b61b123a75b74e32da5 is 50, key is test_row_0/C:col10/1733834057701/Put/seqid=0 2024-12-10T12:34:20,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742034_1210 (size=12301) 2024-12-10T12:34:20,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/91cdb9760da24b61b123a75b74e32da5 2024-12-10T12:34:20,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/9dfff2794b97421b905511781e1ab019 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/9dfff2794b97421b905511781e1ab019 2024-12-10T12:34:20,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/9dfff2794b97421b905511781e1ab019, entries=150, sequenceid=291, filesize=30.5 K 2024-12-10T12:34:20,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/6ba394992e7741f999047f06d2066836 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/6ba394992e7741f999047f06d2066836 2024-12-10T12:34:20,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/6ba394992e7741f999047f06d2066836, entries=150, sequenceid=291, filesize=12.0 K 2024-12-10T12:34:20,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/91cdb9760da24b61b123a75b74e32da5 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/91cdb9760da24b61b123a75b74e32da5 2024-12-10T12:34:20,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/91cdb9760da24b61b123a75b74e32da5, entries=150, sequenceid=291, filesize=12.0 K 2024-12-10T12:34:20,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1267ms, sequenceid=291, compaction requested=true 2024-12-10T12:34:20,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:20,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:20,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:20,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:20,083 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:20,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:20,083 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:20,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:20,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:20,086 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:20,086 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:20,086 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/B is initiating minor compaction (all files) 2024-12-10T12:34:20,086 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/A is initiating minor compaction (all files) 2024-12-10T12:34:20,086 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/B in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:20,086 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/A in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:20,086 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/3ca2c616f53a4499b0f1f31d1fbcfb3d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/82bd985d29914a39946389528c639258, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/2e13a3ad5568478e87553427dedb064e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/6ba394992e7741f999047f06d2066836] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=48.3 K 2024-12-10T12:34:20,086 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e58eaddd2a89454286d72e48a6669cd7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/632eb92b4146422b9ddcf8632aac8d6f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/bd8446786c124f62af48dfe191abf728, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/9dfff2794b97421b905511781e1ab019] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=122.3 K 2024-12-10T12:34:20,086 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:20,086 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e58eaddd2a89454286d72e48a6669cd7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/632eb92b4146422b9ddcf8632aac8d6f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/bd8446786c124f62af48dfe191abf728, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/9dfff2794b97421b905511781e1ab019] 2024-12-10T12:34:20,087 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting e58eaddd2a89454286d72e48a6669cd7, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733834052335 2024-12-10T12:34:20,087 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 3ca2c616f53a4499b0f1f31d1fbcfb3d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733834052335 2024-12-10T12:34:20,087 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 82bd985d29914a39946389528c639258, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733834053449 2024-12-10T12:34:20,087 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 632eb92b4146422b9ddcf8632aac8d6f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733834053449 2024-12-10T12:34:20,088 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2e13a3ad5568478e87553427dedb064e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733834055581 2024-12-10T12:34:20,088 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting bd8446786c124f62af48dfe191abf728, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733834055581 2024-12-10T12:34:20,088 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 6ba394992e7741f999047f06d2066836, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834057701 2024-12-10T12:34:20,089 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9dfff2794b97421b905511781e1ab019, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834057701 2024-12-10T12:34:20,098 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#B#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:20,098 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/7eff3703c85744d983f3019825f7b127 is 50, key is test_row_0/B:col10/1733834057701/Put/seqid=0 2024-12-10T12:34:20,101 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:20,108 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412106900d60677494bc3b96e5b8dd5877558_d32afc9a2cd1759fc0fd2b0a98d70e22 store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:20,109 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412106900d60677494bc3b96e5b8dd5877558_d32afc9a2cd1759fc0fd2b0a98d70e22, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:20,109 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106900d60677494bc3b96e5b8dd5877558_d32afc9a2cd1759fc0fd2b0a98d70e22 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:20,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742035_1211 (size=12949) 2024-12-10T12:34:20,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742036_1212 (size=4469) 2024-12-10T12:34:20,176 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:20,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=64 2024-12-10T12:34:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:20,177 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-10T12:34:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:20,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100164651a2af94942bc2bd3b15e7ce79d_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834058847/Put/seqid=0 2024-12-10T12:34:20,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742037_1213 (size=12454) 2024-12-10T12:34:20,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:20,193 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100164651a2af94942bc2bd3b15e7ce79d_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100164651a2af94942bc2bd3b15e7ce79d_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:20,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/e2e74259f62f421080c726d2e985b078, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:20,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/e2e74259f62f421080c726d2e985b078 is 175, key is test_row_0/A:col10/1733834058847/Put/seqid=0 2024-12-10T12:34:20,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742038_1214 (size=31255) 2024-12-10T12:34:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-10T12:34:20,522 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#A#compaction#175 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:20,522 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/7eff3703c85744d983f3019825f7b127 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7eff3703c85744d983f3019825f7b127 2024-12-10T12:34:20,522 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/49b452dc66f2498e8da3c749b006d237 is 175, key is test_row_0/A:col10/1733834057701/Put/seqid=0 2024-12-10T12:34:20,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742039_1215 (size=31903) 2024-12-10T12:34:20,528 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/B of d32afc9a2cd1759fc0fd2b0a98d70e22 into 7eff3703c85744d983f3019825f7b127(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:20,528 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:20,528 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/B, priority=12, startTime=1733834060083; duration=0sec 2024-12-10T12:34:20,528 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:20,528 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:B 2024-12-10T12:34:20,529 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:20,530 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:20,530 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/C is initiating minor compaction (all files) 2024-12-10T12:34:20,530 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/C in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:20,530 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/81e8da9bf06148f48c4cbe9b2f81070b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/0bc3ec2b074f4274a515c63f3379138a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6a7a9f1b3e2e4575970745973e62b1f9, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/91cdb9760da24b61b123a75b74e32da5] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=48.3 K 2024-12-10T12:34:20,530 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 81e8da9bf06148f48c4cbe9b2f81070b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733834052335 2024-12-10T12:34:20,530 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 0bc3ec2b074f4274a515c63f3379138a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733834053449 2024-12-10T12:34:20,531 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 6a7a9f1b3e2e4575970745973e62b1f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733834055581 2024-12-10T12:34:20,531 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 91cdb9760da24b61b123a75b74e32da5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834057701 2024-12-10T12:34:20,532 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/49b452dc66f2498e8da3c749b006d237 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/49b452dc66f2498e8da3c749b006d237 2024-12-10T12:34:20,537 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/A of d32afc9a2cd1759fc0fd2b0a98d70e22 into 49b452dc66f2498e8da3c749b006d237(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:20,537 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:20,537 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/A, priority=12, startTime=1733834060083; duration=0sec 2024-12-10T12:34:20,537 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:20,537 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:A 2024-12-10T12:34:20,539 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:20,539 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/23332a91e35345d2a68be0e20f2285d2 is 50, key is test_row_0/C:col10/1733834057701/Put/seqid=0 2024-12-10T12:34:20,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742040_1216 (size=12949) 2024-12-10T12:34:20,562 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/23332a91e35345d2a68be0e20f2285d2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/23332a91e35345d2a68be0e20f2285d2 2024-12-10T12:34:20,567 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/C of d32afc9a2cd1759fc0fd2b0a98d70e22 into 23332a91e35345d2a68be0e20f2285d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:20,567 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:20,567 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/C, priority=12, startTime=1733834060083; duration=0sec 2024-12-10T12:34:20,567 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:20,567 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:C 2024-12-10T12:34:20,601 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/e2e74259f62f421080c726d2e985b078 2024-12-10T12:34:20,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/54328367f593482baf4bd9b27dfad0ba is 50, key is test_row_0/B:col10/1733834058847/Put/seqid=0 2024-12-10T12:34:20,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742041_1217 (size=12301) 2024-12-10T12:34:20,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:20,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834120981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:20,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834120982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,017 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/54328367f593482baf4bd9b27dfad0ba 2024-12-10T12:34:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/6331465c01ca4428a03a409de3d53a4d is 50, key is test_row_0/C:col10/1733834058847/Put/seqid=0 2024-12-10T12:34:21,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742042_1218 (size=12301) 2024-12-10T12:34:21,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834121083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834121084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834121286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834121287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,430 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/6331465c01ca4428a03a409de3d53a4d 2024-12-10T12:34:21,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/e2e74259f62f421080c726d2e985b078 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e2e74259f62f421080c726d2e985b078 2024-12-10T12:34:21,437 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e2e74259f62f421080c726d2e985b078, entries=150, sequenceid=317, filesize=30.5 K 2024-12-10T12:34:21,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/54328367f593482baf4bd9b27dfad0ba as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/54328367f593482baf4bd9b27dfad0ba 2024-12-10T12:34:21,441 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/54328367f593482baf4bd9b27dfad0ba, entries=150, sequenceid=317, filesize=12.0 K 2024-12-10T12:34:21,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/6331465c01ca4428a03a409de3d53a4d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6331465c01ca4428a03a409de3d53a4d 2024-12-10T12:34:21,446 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6331465c01ca4428a03a409de3d53a4d, entries=150, sequenceid=317, filesize=12.0 K 2024-12-10T12:34:21,451 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1274ms, sequenceid=317, compaction requested=false 2024-12-10T12:34:21,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:21,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:21,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-12-10T12:34:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=64 2024-12-10T12:34:21,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-12-10T12:34:21,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0360 sec 2024-12-10T12:34:21,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=63, table=TestAcidGuarantees in 2.0390 sec 2024-12-10T12:34:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-10T12:34:21,519 INFO [Thread-667 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 63 completed 2024-12-10T12:34:21,520 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=65, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=65, table=TestAcidGuarantees 2024-12-10T12:34:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-10T12:34:21,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=65, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=65, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:21,522 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=65, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=65, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:21,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:21,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:34:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:21,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121048d020227a1e4f908e689f9ca33572c3_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834060981/Put/seqid=0 2024-12-10T12:34:21,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742043_1219 (size=14994) 2024-12-10T12:34:21,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:21,603 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121048d020227a1e4f908e689f9ca33572c3_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121048d020227a1e4f908e689f9ca33572c3_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:21,604 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/b1931a53af1d4ef383ea108b1f9f4bd6, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:21,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/b1931a53af1d4ef383ea108b1f9f4bd6 is 175, key is test_row_0/A:col10/1733834060981/Put/seqid=0 2024-12-10T12:34:21,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742044_1220 (size=39949) 2024-12-10T12:34:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-10T12:34:21,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834121632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834121632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,658 DEBUG [Thread-668 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x517ff977 to 127.0.0.1:64063 2024-12-10T12:34:21,658 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:21,659 DEBUG [Thread-670 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3448d233 to 127.0.0.1:64063 2024-12-10T12:34:21,659 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:21,660 DEBUG [Thread-674 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08a7e1dd to 127.0.0.1:64063 2024-12-10T12:34:21,660 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:21,663 DEBUG [Thread-672 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a11164b to 127.0.0.1:64063 2024-12-10T12:34:21,663 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:21,674 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:21,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:21,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:21,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:21,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:21,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(58): pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:21,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:21,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834121735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834121735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-10T12:34:21,826 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:21,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:21,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:21,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:21,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:21,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(58): pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:21,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:21,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834121938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834121939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:21,983 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:21,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:21,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(58): pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,009 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/b1931a53af1d4ef383ea108b1f9f4bd6 2024-12-10T12:34:22,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/f89e4255fa0841f38b6cf4dc53182075 is 50, key is test_row_0/B:col10/1733834060981/Put/seqid=0 2024-12-10T12:34:22,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742045_1221 (size=12301) 2024-12-10T12:34:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-10T12:34:22,140 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:22,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:22,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:22,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(58): pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834122243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:22,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834122243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:22,292 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:22,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(58): pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,445 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:22,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:22,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(58): pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/f89e4255fa0841f38b6cf4dc53182075 2024-12-10T12:34:22,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/bc0edf0a926449f39c841ed0f161a4be is 50, key is test_row_0/C:col10/1733834060981/Put/seqid=0 2024-12-10T12:34:22,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742046_1222 (size=12301) 2024-12-10T12:34:22,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1733834122569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:22,571 DEBUG [Thread-665 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=8, retries=16, started=18218 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:34:22,599 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:22,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(58): pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-10T12:34:22,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43274 deadline: 1733834122746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:22,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43330 deadline: 1733834122748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 2024-12-10T12:34:22,755 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:22,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(58): pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/bc0edf0a926449f39c841ed0f161a4be 2024-12-10T12:34:22,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/b1931a53af1d4ef383ea108b1f9f4bd6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b1931a53af1d4ef383ea108b1f9f4bd6 2024-12-10T12:34:22,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b1931a53af1d4ef383ea108b1f9f4bd6, entries=200, sequenceid=332, filesize=39.0 K 2024-12-10T12:34:22,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/f89e4255fa0841f38b6cf4dc53182075 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/f89e4255fa0841f38b6cf4dc53182075 2024-12-10T12:34:22,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/f89e4255fa0841f38b6cf4dc53182075, entries=150, sequenceid=332, filesize=12.0 K 2024-12-10T12:34:22,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/bc0edf0a926449f39c841ed0f161a4be as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc0edf0a926449f39c841ed0f161a4be 2024-12-10T12:34:22,908 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:22,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(58): pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:22,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc0edf0a926449f39c841ed0f161a4be, entries=150, sequenceid=332, filesize=12.0 K 2024-12-10T12:34:22,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1322ms, sequenceid=332, compaction requested=true 2024-12-10T12:34:22,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:22,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:22,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:22,912 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:22,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:22,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:22,913 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:22,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d32afc9a2cd1759fc0fd2b0a98d70e22:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:22,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:22,914 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:22,914 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:22,914 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/A is initiating minor compaction (all files) 2024-12-10T12:34:22,914 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/B is initiating minor compaction (all files) 2024-12-10T12:34:22,914 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/A in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,914 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/B in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,914 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/49b452dc66f2498e8da3c749b006d237, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e2e74259f62f421080c726d2e985b078, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b1931a53af1d4ef383ea108b1f9f4bd6] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=100.7 K 2024-12-10T12:34:22,914 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:22,914 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7eff3703c85744d983f3019825f7b127, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/54328367f593482baf4bd9b27dfad0ba, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/f89e4255fa0841f38b6cf4dc53182075] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=36.7 K 2024-12-10T12:34:22,914 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/49b452dc66f2498e8da3c749b006d237, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e2e74259f62f421080c726d2e985b078, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b1931a53af1d4ef383ea108b1f9f4bd6] 2024-12-10T12:34:22,915 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 7eff3703c85744d983f3019825f7b127, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834057701 2024-12-10T12:34:22,915 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 49b452dc66f2498e8da3c749b006d237, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834057701 2024-12-10T12:34:22,915 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting e2e74259f62f421080c726d2e985b078, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733834058847 2024-12-10T12:34:22,915 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 54328367f593482baf4bd9b27dfad0ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733834058847 2024-12-10T12:34:22,916 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting b1931a53af1d4ef383ea108b1f9f4bd6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733834060980 2024-12-10T12:34:22,916 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting f89e4255fa0841f38b6cf4dc53182075, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733834060981 2024-12-10T12:34:22,926 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#B#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:22,927 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/edfb252f2af84341a14f5c502228274e is 50, key is test_row_0/B:col10/1733834060981/Put/seqid=0 2024-12-10T12:34:22,928 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:22,938 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210a1c76b2f4079431c976c0806a8202dc9_d32afc9a2cd1759fc0fd2b0a98d70e22 store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:22,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742047_1223 (size=13051) 2024-12-10T12:34:22,965 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210a1c76b2f4079431c976c0806a8202dc9_d32afc9a2cd1759fc0fd2b0a98d70e22, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:22,966 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a1c76b2f4079431c976c0806a8202dc9_d32afc9a2cd1759fc0fd2b0a98d70e22 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:22,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742048_1224 (size=4469) 2024-12-10T12:34:23,061 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:23,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-12-10T12:34:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:23,062 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:34:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:23,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:23,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:23,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:23,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cec2451efd504ba89130c49622303402_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834061629/Put/seqid=0 2024-12-10T12:34:23,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742049_1225 (size=12454) 2024-12-10T12:34:23,349 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/edfb252f2af84341a14f5c502228274e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/edfb252f2af84341a14f5c502228274e 2024-12-10T12:34:23,353 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/B of d32afc9a2cd1759fc0fd2b0a98d70e22 into edfb252f2af84341a14f5c502228274e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:23,353 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:23,353 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/B, priority=13, startTime=1733834062912; duration=0sec 2024-12-10T12:34:23,354 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:23,354 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:B 2024-12-10T12:34:23,354 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:23,355 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:23,355 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): d32afc9a2cd1759fc0fd2b0a98d70e22/C is initiating minor compaction (all files) 2024-12-10T12:34:23,355 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d32afc9a2cd1759fc0fd2b0a98d70e22/C in TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:23,355 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/23332a91e35345d2a68be0e20f2285d2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6331465c01ca4428a03a409de3d53a4d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc0edf0a926449f39c841ed0f161a4be] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp, totalSize=36.7 K 2024-12-10T12:34:23,355 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 23332a91e35345d2a68be0e20f2285d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733834057701 2024-12-10T12:34:23,355 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 6331465c01ca4428a03a409de3d53a4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733834058847 2024-12-10T12:34:23,356 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting bc0edf0a926449f39c841ed0f161a4be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733834060981 2024-12-10T12:34:23,362 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:23,362 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/374a52cd54f544a394965dedd2ba2a1b is 50, key is test_row_0/C:col10/1733834060981/Put/seqid=0 2024-12-10T12:34:23,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742050_1226 (size=13051) 2024-12-10T12:34:23,371 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d32afc9a2cd1759fc0fd2b0a98d70e22#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:23,371 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/c2d7104b407a4e87bd8f3e4d0d1e0ec4 is 175, key is test_row_0/A:col10/1733834060981/Put/seqid=0 2024-12-10T12:34:23,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742051_1227 (size=32005) 2024-12-10T12:34:23,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:23,486 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cec2451efd504ba89130c49622303402_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cec2451efd504ba89130c49622303402_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/52b51c8fe92141ada2d83097dab9ee0c, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:23,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/52b51c8fe92141ada2d83097dab9ee0c is 175, key is test_row_0/A:col10/1733834061629/Put/seqid=0 2024-12-10T12:34:23,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742052_1228 (size=31255) 2024-12-10T12:34:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-10T12:34:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. as already flushing 2024-12-10T12:34:23,719 DEBUG [Thread-659 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x176c5c1b to 127.0.0.1:64063 2024-12-10T12:34:23,719 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:23,735 DEBUG [Thread-657 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x433e2b26 to 127.0.0.1:64063 2024-12-10T12:34:23,736 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:23,751 DEBUG [Thread-661 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x24f64590 to 127.0.0.1:64063 2024-12-10T12:34:23,751 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:23,767 DEBUG [Thread-663 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7c5c4716 to 127.0.0.1:64063 2024-12-10T12:34:23,767 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:23,773 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/374a52cd54f544a394965dedd2ba2a1b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/374a52cd54f544a394965dedd2ba2a1b 2024-12-10T12:34:23,781 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/C of d32afc9a2cd1759fc0fd2b0a98d70e22 into 374a52cd54f544a394965dedd2ba2a1b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:23,781 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:23,781 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/C, priority=13, startTime=1733834062913; duration=0sec 2024-12-10T12:34:23,781 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:23,781 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:C 2024-12-10T12:34:23,782 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/c2d7104b407a4e87bd8f3e4d0d1e0ec4 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c2d7104b407a4e87bd8f3e4d0d1e0ec4 2024-12-10T12:34:23,786 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d32afc9a2cd1759fc0fd2b0a98d70e22/A of d32afc9a2cd1759fc0fd2b0a98d70e22 into c2d7104b407a4e87bd8f3e4d0d1e0ec4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:23,787 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:23,787 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22., storeName=d32afc9a2cd1759fc0fd2b0a98d70e22/A, priority=13, startTime=1733834062912; duration=0sec 2024-12-10T12:34:23,787 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:23,787 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d32afc9a2cd1759fc0fd2b0a98d70e22:A 2024-12-10T12:34:23,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/52b51c8fe92141ada2d83097dab9ee0c 2024-12-10T12:34:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/8359a81450434a21a916ce090ba24043 is 50, key is test_row_0/B:col10/1733834061629/Put/seqid=0 2024-12-10T12:34:23,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742053_1229 (size=12301) 2024-12-10T12:34:24,315 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/8359a81450434a21a916ce090ba24043 2024-12-10T12:34:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/9edf267e88ba474b812bf6ef15410d34 is 50, key is test_row_0/C:col10/1733834061629/Put/seqid=0 2024-12-10T12:34:24,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742054_1230 (size=12301) 2024-12-10T12:34:24,732 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/9edf267e88ba474b812bf6ef15410d34 2024-12-10T12:34:24,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/52b51c8fe92141ada2d83097dab9ee0c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/52b51c8fe92141ada2d83097dab9ee0c 2024-12-10T12:34:24,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/52b51c8fe92141ada2d83097dab9ee0c, entries=150, sequenceid=357, filesize=30.5 K 2024-12-10T12:34:24,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/8359a81450434a21a916ce090ba24043 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/8359a81450434a21a916ce090ba24043 2024-12-10T12:34:24,745 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/8359a81450434a21a916ce090ba24043, entries=150, sequenceid=357, filesize=12.0 K 2024-12-10T12:34:24,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/9edf267e88ba474b812bf6ef15410d34 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/9edf267e88ba474b812bf6ef15410d34 2024-12-10T12:34:24,750 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/9edf267e88ba474b812bf6ef15410d34, entries=150, sequenceid=357, filesize=12.0 K 2024-12-10T12:34:24,750 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1688ms, sequenceid=357, compaction requested=false 2024-12-10T12:34:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2538): Flush status journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-10T12:34:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=66 2024-12-10T12:34:24,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=66, resume processing ppid=65 2024-12-10T12:34:24,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, ppid=65, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2290 sec 2024-12-10T12:34:24,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=65, table=TestAcidGuarantees in 3.2330 sec 2024-12-10T12:34:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-10T12:34:25,630 INFO [Thread-667 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 65 completed 2024-12-10T12:34:26,112 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T12:34:31,436 DEBUG [master/fad905551524:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region bab6b7e921719ae441e5ef7d31c45c53 changed from -1.0 to 0.0, refreshing cache 2024-12-10T12:34:32,664 DEBUG [Thread-665 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x68f0be85 to 127.0.0.1:64063 2024-12-10T12:34:32,664 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 8973 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 8843 2024-12-10T12:34:32,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T12:34:32,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3756 2024-12-10T12:34:32,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 11266 rows 2024-12-10T12:34:32,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3747 2024-12-10T12:34:32,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 11238 rows 2024-12-10T12:34:32,666 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-10T12:34:32,666 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7cca453a to 127.0.0.1:64063 2024-12-10T12:34:32,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:32,671 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-12-10T12:34:32,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T12:34:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=67, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T12:34:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-10T12:34:32,674 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834072674"}]},"ts":"1733834072674"} 2024-12-10T12:34:32,675 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T12:34:32,733 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T12:34:32,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T12:34:32,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, UNASSIGN}] 2024-12-10T12:34:32,737 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, UNASSIGN 2024-12-10T12:34:32,739 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=d32afc9a2cd1759fc0fd2b0a98d70e22, regionState=CLOSING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:34:32,740 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T12:34:32,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; CloseRegionProcedure d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617}] 2024-12-10T12:34:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-10T12:34:32,893 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:32,894 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(124): Close d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:32,894 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-12-10T12:34:32,895 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1681): Closing d32afc9a2cd1759fc0fd2b0a98d70e22, disabling compactions & flushes 2024-12-10T12:34:32,895 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:32,895 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:32,895 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. after waiting 0 ms 2024-12-10T12:34:32,895 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:32,896 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(2837): Flushing d32afc9a2cd1759fc0fd2b0a98d70e22 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-10T12:34:32,896 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=A 2024-12-10T12:34:32,896 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:32,897 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=B 2024-12-10T12:34:32,897 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:32,897 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d32afc9a2cd1759fc0fd2b0a98d70e22, store=C 2024-12-10T12:34:32,897 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:32,906 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106767c50aca4f49309f50efbd9a74ce66_d32afc9a2cd1759fc0fd2b0a98d70e22 is 50, key is test_row_0/A:col10/1733834072660/Put/seqid=0 2024-12-10T12:34:32,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742055_1231 (size=9914) 2024-12-10T12:34:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-10T12:34:33,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-10T12:34:33,313 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:33,323 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106767c50aca4f49309f50efbd9a74ce66_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106767c50aca4f49309f50efbd9a74ce66_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:33,326 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/8f2a51067f564dbcbaea453c281eb9ac, store: [table=TestAcidGuarantees family=A region=d32afc9a2cd1759fc0fd2b0a98d70e22] 2024-12-10T12:34:33,327 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/8f2a51067f564dbcbaea453c281eb9ac is 175, key is test_row_0/A:col10/1733834072660/Put/seqid=0 2024-12-10T12:34:33,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742056_1232 (size=22561) 2024-12-10T12:34:33,734 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/8f2a51067f564dbcbaea453c281eb9ac 2024-12-10T12:34:33,747 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/3fe93232d31e4062b6e6475650ed28b1 is 50, key is test_row_0/B:col10/1733834072660/Put/seqid=0 2024-12-10T12:34:33,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742057_1233 (size=9857) 2024-12-10T12:34:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-10T12:34:34,153 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/3fe93232d31e4062b6e6475650ed28b1 2024-12-10T12:34:34,165 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/05f230888e9c4925819940417082bf14 is 50, key is test_row_0/C:col10/1733834072660/Put/seqid=0 2024-12-10T12:34:34,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742058_1234 (size=9857) 2024-12-10T12:34:34,572 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/05f230888e9c4925819940417082bf14 2024-12-10T12:34:34,583 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/A/8f2a51067f564dbcbaea453c281eb9ac as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8f2a51067f564dbcbaea453c281eb9ac 2024-12-10T12:34:34,589 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8f2a51067f564dbcbaea453c281eb9ac, entries=100, sequenceid=368, filesize=22.0 K 2024-12-10T12:34:34,590 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/B/3fe93232d31e4062b6e6475650ed28b1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/3fe93232d31e4062b6e6475650ed28b1 2024-12-10T12:34:34,594 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/3fe93232d31e4062b6e6475650ed28b1, entries=100, sequenceid=368, filesize=9.6 K 2024-12-10T12:34:34,595 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/.tmp/C/05f230888e9c4925819940417082bf14 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/05f230888e9c4925819940417082bf14 2024-12-10T12:34:34,600 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/05f230888e9c4925819940417082bf14, entries=100, sequenceid=368, filesize=9.6 K 2024-12-10T12:34:34,601 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(3040): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d32afc9a2cd1759fc0fd2b0a98d70e22 in 1706ms, sequenceid=368, compaction requested=true 2024-12-10T12:34:34,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8c1d4f89020f4cc8b7fb3056b50e9c93, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2e0205ccf70a47329b81d443e008148f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f366a8d71338428cb71d9603206fdbf9, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b0eb84cb7301490b859f98c4d4bf9fae, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c5a12f966b384043ad3c33fa954efe9d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2ee481253bd449b8878dfdb5711b69f0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/fb16fa8141db47d48633b0befed1fa0e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f13ee1f8a78842c3b090c592f75851d1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/99b5bbf893d5463f979e686870161cbb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/dba5b60a005d4e2f87b3d9f513a8d58e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/0e9f0cee67ff45cea2326c7205bbe0b2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f61a243cc6af4118a26d3939da26153a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/407d26c1b12c4a478faa2631217cfc44, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/6e6c0ed3037e430bb02a508ecc1f582d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/68d20620347d48a299c3147d17cc9c97, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e58eaddd2a89454286d72e48a6669cd7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/632eb92b4146422b9ddcf8632aac8d6f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/bd8446786c124f62af48dfe191abf728, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/49b452dc66f2498e8da3c749b006d237, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/9dfff2794b97421b905511781e1ab019, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e2e74259f62f421080c726d2e985b078, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b1931a53af1d4ef383ea108b1f9f4bd6] to archive 2024-12-10T12:34:34,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:34:34,605 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2e0205ccf70a47329b81d443e008148f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2e0205ccf70a47329b81d443e008148f 2024-12-10T12:34:34,605 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8c1d4f89020f4cc8b7fb3056b50e9c93 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8c1d4f89020f4cc8b7fb3056b50e9c93 2024-12-10T12:34:34,605 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f366a8d71338428cb71d9603206fdbf9 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f366a8d71338428cb71d9603206fdbf9 2024-12-10T12:34:34,605 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b0eb84cb7301490b859f98c4d4bf9fae to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b0eb84cb7301490b859f98c4d4bf9fae 2024-12-10T12:34:34,606 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/fb16fa8141db47d48633b0befed1fa0e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/fb16fa8141db47d48633b0befed1fa0e 2024-12-10T12:34:34,606 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2ee481253bd449b8878dfdb5711b69f0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/2ee481253bd449b8878dfdb5711b69f0 2024-12-10T12:34:34,606 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c5a12f966b384043ad3c33fa954efe9d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c5a12f966b384043ad3c33fa954efe9d 2024-12-10T12:34:34,606 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f13ee1f8a78842c3b090c592f75851d1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f13ee1f8a78842c3b090c592f75851d1 2024-12-10T12:34:34,607 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/dba5b60a005d4e2f87b3d9f513a8d58e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/dba5b60a005d4e2f87b3d9f513a8d58e 2024-12-10T12:34:34,607 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/0e9f0cee67ff45cea2326c7205bbe0b2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/0e9f0cee67ff45cea2326c7205bbe0b2 2024-12-10T12:34:34,607 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f61a243cc6af4118a26d3939da26153a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/f61a243cc6af4118a26d3939da26153a 2024-12-10T12:34:34,607 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/99b5bbf893d5463f979e686870161cbb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/99b5bbf893d5463f979e686870161cbb 2024-12-10T12:34:34,608 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/407d26c1b12c4a478faa2631217cfc44 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/407d26c1b12c4a478faa2631217cfc44 2024-12-10T12:34:34,608 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/6e6c0ed3037e430bb02a508ecc1f582d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/6e6c0ed3037e430bb02a508ecc1f582d 2024-12-10T12:34:34,608 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e58eaddd2a89454286d72e48a6669cd7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e58eaddd2a89454286d72e48a6669cd7 2024-12-10T12:34:34,608 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/68d20620347d48a299c3147d17cc9c97 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/68d20620347d48a299c3147d17cc9c97 2024-12-10T12:34:34,608 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/632eb92b4146422b9ddcf8632aac8d6f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/632eb92b4146422b9ddcf8632aac8d6f 2024-12-10T12:34:34,609 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/bd8446786c124f62af48dfe191abf728 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/bd8446786c124f62af48dfe191abf728 2024-12-10T12:34:34,609 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/49b452dc66f2498e8da3c749b006d237 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/49b452dc66f2498e8da3c749b006d237 2024-12-10T12:34:34,609 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/9dfff2794b97421b905511781e1ab019 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/9dfff2794b97421b905511781e1ab019 2024-12-10T12:34:34,609 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e2e74259f62f421080c726d2e985b078 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/e2e74259f62f421080c726d2e985b078 2024-12-10T12:34:34,609 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b1931a53af1d4ef383ea108b1f9f4bd6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/b1931a53af1d4ef383ea108b1f9f4bd6 2024-12-10T12:34:34,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/a80d3d51978943638b8975b805f5a3bf, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/204a1483ef604b7d90b26a860e142c6c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d19a5feed7b5401895bdde4e62d2ed15, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/615adadbf6844ef5aeb826df4b976197, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7d440809a4884eddb3563f4d2f10b594, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d8ada2b2dd114f03894cdc7ce15d38e6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/664d4694272c41718f7a8549bda502da, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b5725f4ff3064dc488ea2eb2a59c4eb8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/862014d977a14157a69b8c209662ee60, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/5f6ea613e1384ce6b700de21aed177a2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/bd3bf922ed324767aadbf773060b4385, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/52f1377349714408936e841d23459c5c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/20d31a2826c84689aab19eb6223a8e4b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b139d869e093470a92c643703403cd63, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/3ca2c616f53a4499b0f1f31d1fbcfb3d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/ea043f003a5d4869882f946516b98261, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/82bd985d29914a39946389528c639258, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/2e13a3ad5568478e87553427dedb064e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7eff3703c85744d983f3019825f7b127, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/6ba394992e7741f999047f06d2066836, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/54328367f593482baf4bd9b27dfad0ba, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/f89e4255fa0841f38b6cf4dc53182075] to archive 2024-12-10T12:34:34,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:34:34,613 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d19a5feed7b5401895bdde4e62d2ed15 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d19a5feed7b5401895bdde4e62d2ed15 2024-12-10T12:34:34,613 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/a80d3d51978943638b8975b805f5a3bf to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/a80d3d51978943638b8975b805f5a3bf 2024-12-10T12:34:34,613 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7d440809a4884eddb3563f4d2f10b594 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7d440809a4884eddb3563f4d2f10b594 2024-12-10T12:34:34,613 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/204a1483ef604b7d90b26a860e142c6c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/204a1483ef604b7d90b26a860e142c6c 2024-12-10T12:34:34,613 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/664d4694272c41718f7a8549bda502da to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/664d4694272c41718f7a8549bda502da 2024-12-10T12:34:34,613 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b5725f4ff3064dc488ea2eb2a59c4eb8 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b5725f4ff3064dc488ea2eb2a59c4eb8 2024-12-10T12:34:34,613 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d8ada2b2dd114f03894cdc7ce15d38e6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/d8ada2b2dd114f03894cdc7ce15d38e6 2024-12-10T12:34:34,614 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/615adadbf6844ef5aeb826df4b976197 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/615adadbf6844ef5aeb826df4b976197 2024-12-10T12:34:34,615 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/bd3bf922ed324767aadbf773060b4385 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/bd3bf922ed324767aadbf773060b4385 2024-12-10T12:34:34,615 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/5f6ea613e1384ce6b700de21aed177a2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/5f6ea613e1384ce6b700de21aed177a2 2024-12-10T12:34:34,615 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/20d31a2826c84689aab19eb6223a8e4b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/20d31a2826c84689aab19eb6223a8e4b 2024-12-10T12:34:34,615 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/3ca2c616f53a4499b0f1f31d1fbcfb3d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/3ca2c616f53a4499b0f1f31d1fbcfb3d 2024-12-10T12:34:34,615 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/52f1377349714408936e841d23459c5c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/52f1377349714408936e841d23459c5c 2024-12-10T12:34:34,615 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/862014d977a14157a69b8c209662ee60 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/862014d977a14157a69b8c209662ee60 2024-12-10T12:34:34,615 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b139d869e093470a92c643703403cd63 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/b139d869e093470a92c643703403cd63 2024-12-10T12:34:34,615 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/ea043f003a5d4869882f946516b98261 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/ea043f003a5d4869882f946516b98261 2024-12-10T12:34:34,616 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/82bd985d29914a39946389528c639258 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/82bd985d29914a39946389528c639258 2024-12-10T12:34:34,616 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/6ba394992e7741f999047f06d2066836 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/6ba394992e7741f999047f06d2066836 2024-12-10T12:34:34,616 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/f89e4255fa0841f38b6cf4dc53182075 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/f89e4255fa0841f38b6cf4dc53182075 2024-12-10T12:34:34,617 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/2e13a3ad5568478e87553427dedb064e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/2e13a3ad5568478e87553427dedb064e 2024-12-10T12:34:34,617 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/54328367f593482baf4bd9b27dfad0ba to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/54328367f593482baf4bd9b27dfad0ba 2024-12-10T12:34:34,617 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7eff3703c85744d983f3019825f7b127 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/7eff3703c85744d983f3019825f7b127 2024-12-10T12:34:34,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/1627927ff52a4e06aee0efa2d1bd06d7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5e6fcd0092064fb9a8a2ef8c26cc5f4a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/be7c1dd5d80a40c2b767238d19bf2fcf, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/2f52a85d63c44b1faccd7c503b6eaa6b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/c5fb5b4ff9214ee5a9e115fbb306dca3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/45cf39fc38d045958d7ef46a36f08830, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/e89d0b6fc99b41e6b5da65a55574245d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/996b3d8a499744928fbe156a10a7c26f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/74e74b31a6804134befe0e20d5f23ce2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc5503bb8d7e4738ad6a9f4449e5a033, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5317a82b323047f9b5ee0458fb0e02fe, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/8bfd65e86b754f77b627a74f78d9d1b6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/a80f562fb84140daa0497c914ae45989, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/66f6d4b27e694830af2b6ee145b02dc6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/81e8da9bf06148f48c4cbe9b2f81070b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/3c5993b0bf0043d0b165b607d15df152, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/0bc3ec2b074f4274a515c63f3379138a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6a7a9f1b3e2e4575970745973e62b1f9, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/23332a91e35345d2a68be0e20f2285d2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/91cdb9760da24b61b123a75b74e32da5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6331465c01ca4428a03a409de3d53a4d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc0edf0a926449f39c841ed0f161a4be] to archive 2024-12-10T12:34:34,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:34:34,620 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/be7c1dd5d80a40c2b767238d19bf2fcf to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/be7c1dd5d80a40c2b767238d19bf2fcf 2024-12-10T12:34:34,620 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5e6fcd0092064fb9a8a2ef8c26cc5f4a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5e6fcd0092064fb9a8a2ef8c26cc5f4a 2024-12-10T12:34:34,620 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/1627927ff52a4e06aee0efa2d1bd06d7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/1627927ff52a4e06aee0efa2d1bd06d7 2024-12-10T12:34:34,620 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/2f52a85d63c44b1faccd7c503b6eaa6b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/2f52a85d63c44b1faccd7c503b6eaa6b 2024-12-10T12:34:34,620 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/e89d0b6fc99b41e6b5da65a55574245d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/e89d0b6fc99b41e6b5da65a55574245d 2024-12-10T12:34:34,620 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/996b3d8a499744928fbe156a10a7c26f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/996b3d8a499744928fbe156a10a7c26f 2024-12-10T12:34:34,620 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/45cf39fc38d045958d7ef46a36f08830 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/45cf39fc38d045958d7ef46a36f08830 2024-12-10T12:34:34,621 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/c5fb5b4ff9214ee5a9e115fbb306dca3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/c5fb5b4ff9214ee5a9e115fbb306dca3 2024-12-10T12:34:34,622 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/74e74b31a6804134befe0e20d5f23ce2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/74e74b31a6804134befe0e20d5f23ce2 2024-12-10T12:34:34,622 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc5503bb8d7e4738ad6a9f4449e5a033 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc5503bb8d7e4738ad6a9f4449e5a033 2024-12-10T12:34:34,622 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/8bfd65e86b754f77b627a74f78d9d1b6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/8bfd65e86b754f77b627a74f78d9d1b6 2024-12-10T12:34:34,622 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5317a82b323047f9b5ee0458fb0e02fe to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/5317a82b323047f9b5ee0458fb0e02fe 2024-12-10T12:34:34,622 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/a80f562fb84140daa0497c914ae45989 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/a80f562fb84140daa0497c914ae45989 2024-12-10T12:34:34,622 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/3c5993b0bf0043d0b165b607d15df152 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/3c5993b0bf0043d0b165b607d15df152 2024-12-10T12:34:34,622 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/81e8da9bf06148f48c4cbe9b2f81070b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/81e8da9bf06148f48c4cbe9b2f81070b 2024-12-10T12:34:34,622 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/66f6d4b27e694830af2b6ee145b02dc6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/66f6d4b27e694830af2b6ee145b02dc6 2024-12-10T12:34:34,623 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6a7a9f1b3e2e4575970745973e62b1f9 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6a7a9f1b3e2e4575970745973e62b1f9 2024-12-10T12:34:34,623 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/0bc3ec2b074f4274a515c63f3379138a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/0bc3ec2b074f4274a515c63f3379138a 2024-12-10T12:34:34,624 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/91cdb9760da24b61b123a75b74e32da5 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/91cdb9760da24b61b123a75b74e32da5 2024-12-10T12:34:34,624 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc0edf0a926449f39c841ed0f161a4be to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/bc0edf0a926449f39c841ed0f161a4be 2024-12-10T12:34:34,624 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/23332a91e35345d2a68be0e20f2285d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/23332a91e35345d2a68be0e20f2285d2 2024-12-10T12:34:34,624 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6331465c01ca4428a03a409de3d53a4d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/6331465c01ca4428a03a409de3d53a4d 2024-12-10T12:34:34,627 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/recovered.edits/371.seqid, newMaxSeqId=371, maxSeqId=4 2024-12-10T12:34:34,628 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22. 2024-12-10T12:34:34,628 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1635): Region close journal for d32afc9a2cd1759fc0fd2b0a98d70e22: 2024-12-10T12:34:34,629 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(170): Closed d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,630 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=d32afc9a2cd1759fc0fd2b0a98d70e22, regionState=CLOSED 2024-12-10T12:34:34,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-12-10T12:34:34,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; CloseRegionProcedure d32afc9a2cd1759fc0fd2b0a98d70e22, server=fad905551524,38809,1733834007617 in 1.8910 sec 2024-12-10T12:34:34,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=69, resume processing ppid=68 2024-12-10T12:34:34,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d32afc9a2cd1759fc0fd2b0a98d70e22, UNASSIGN in 1.8960 sec 2024-12-10T12:34:34,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-12-10T12:34:34,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8990 sec 2024-12-10T12:34:34,634 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834074634"}]},"ts":"1733834074634"} 2024-12-10T12:34:34,635 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T12:34:34,674 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T12:34:34,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 2.0040 sec 2024-12-10T12:34:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-10T12:34:34,782 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 67 completed 2024-12-10T12:34:34,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T12:34:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=71, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:34:34,786 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=71, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:34:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-10T12:34:34,787 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=71, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:34:34,790 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,796 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/recovered.edits] 2024-12-10T12:34:34,801 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/52b51c8fe92141ada2d83097dab9ee0c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/52b51c8fe92141ada2d83097dab9ee0c 2024-12-10T12:34:34,802 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c2d7104b407a4e87bd8f3e4d0d1e0ec4 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/c2d7104b407a4e87bd8f3e4d0d1e0ec4 2024-12-10T12:34:34,802 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8f2a51067f564dbcbaea453c281eb9ac to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/A/8f2a51067f564dbcbaea453c281eb9ac 2024-12-10T12:34:34,804 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/3fe93232d31e4062b6e6475650ed28b1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/3fe93232d31e4062b6e6475650ed28b1 2024-12-10T12:34:34,804 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/8359a81450434a21a916ce090ba24043 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/8359a81450434a21a916ce090ba24043 2024-12-10T12:34:34,804 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/edfb252f2af84341a14f5c502228274e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/B/edfb252f2af84341a14f5c502228274e 2024-12-10T12:34:34,807 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/374a52cd54f544a394965dedd2ba2a1b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/374a52cd54f544a394965dedd2ba2a1b 2024-12-10T12:34:34,807 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/9edf267e88ba474b812bf6ef15410d34 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/9edf267e88ba474b812bf6ef15410d34 2024-12-10T12:34:34,807 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/05f230888e9c4925819940417082bf14 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/C/05f230888e9c4925819940417082bf14 2024-12-10T12:34:34,810 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/recovered.edits/371.seqid to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22/recovered.edits/371.seqid 2024-12-10T12:34:34,810 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,810 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T12:34:34,810 DEBUG [PEWorker-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T12:34:34,811 DEBUG [PEWorker-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T12:34:34,818 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121006ef9651274d47cebdb072617f081a46_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121006ef9651274d47cebdb072617f081a46_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,818 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100164651a2af94942bc2bd3b15e7ce79d_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100164651a2af94942bc2bd3b15e7ce79d_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,819 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100c0d3654f7a54325820ba1359f335145_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100c0d3654f7a54325820ba1359f335145_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,819 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121016cc5059c3d24f5292cc3132b1b2611b_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121016cc5059c3d24f5292cc3132b1b2611b_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,819 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210195db30f3b8c4c788bbcb59354a82925_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210195db30f3b8c4c788bbcb59354a82925_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,819 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100c761c6b6d0842b09b007e86222408d7_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100c761c6b6d0842b09b007e86222408d7_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,819 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100412ed0802ae40b4b0e1c1920dcdca8c_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100412ed0802ae40b4b0e1c1920dcdca8c_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,819 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210280c1399a383493093953c607f84a51f_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210280c1399a383493093953c607f84a51f_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,820 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121036cf9e05615045cd93782e5ab69beeba_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121036cf9e05615045cd93782e5ab69beeba_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,820 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105dac59b7940a4b42b84ec3fbe5c61d7a_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105dac59b7940a4b42b84ec3fbe5c61d7a_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,820 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106767c50aca4f49309f50efbd9a74ce66_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106767c50aca4f49309f50efbd9a74ce66_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,820 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121076b23084c52344b6ad78aef5d5b9bc83_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121076b23084c52344b6ad78aef5d5b9bc83_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,820 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121068d8215d6d084d0e86ee4be6baab091b_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121068d8215d6d084d0e86ee4be6baab091b_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,821 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bce1d780aed741d18f1f95c67b354f7a_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bce1d780aed741d18f1f95c67b354f7a_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,821 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cc2fad51ab9e488c96a99bf2fb682a22_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cc2fad51ab9e488c96a99bf2fb682a22_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,821 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121048d020227a1e4f908e689f9ca33572c3_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121048d020227a1e4f908e689f9ca33572c3_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,821 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cec2451efd504ba89130c49622303402_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cec2451efd504ba89130c49622303402_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,821 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d7f69a24670d4ef6a3f60e5bbecf241a_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d7f69a24670d4ef6a3f60e5bbecf241a_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,821 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210efa750cbb7f9421ab1c479b14ad5d234_d32afc9a2cd1759fc0fd2b0a98d70e22 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210efa750cbb7f9421ab1c479b14ad5d234_d32afc9a2cd1759fc0fd2b0a98d70e22 2024-12-10T12:34:34,822 DEBUG [PEWorker-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T12:34:34,824 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=71, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:34:34,826 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T12:34:34,828 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T12:34:34,829 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=71, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:34:34,829 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T12:34:34,829 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733834074829"}]},"ts":"9223372036854775807"} 2024-12-10T12:34:34,831 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-12-10T12:34:34,831 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d32afc9a2cd1759fc0fd2b0a98d70e22, NAME => 'TestAcidGuarantees,,1733834038344.d32afc9a2cd1759fc0fd2b0a98d70e22.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T12:34:34,831 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T12:34:34,831 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733834074831"}]},"ts":"9223372036854775807"} 2024-12-10T12:34:34,833 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-12-10T12:34:34,841 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=71, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:34:34,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-12-10T12:34:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-10T12:34:34,889 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 71 completed 2024-12-10T12:34:34,899 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMobMixedAtomicity Thread=244 (was 247), OpenFileDescriptor=445 (was 460), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=243 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=9492 (was 9602) 2024-12-10T12:34:34,905 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testGetAtomicity Thread=244, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=243, ProcessCount=11, AvailableMemoryMB=9492 2024-12-10T12:34:34,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T12:34:34,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:34:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T12:34:34,908 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T12:34:34,908 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:34,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 72 2024-12-10T12:34:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-12-10T12:34:34,909 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T12:34:34,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742059_1235 (size=963) 2024-12-10T12:34:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-12-10T12:34:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-12-10T12:34:35,321 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 2024-12-10T12:34:35,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742060_1236 (size=53) 2024-12-10T12:34:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-12-10T12:34:35,733 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:34:35,733 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing 430faea91f05d6420a9ced51a112393c, disabling compactions & flushes 2024-12-10T12:34:35,733 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:35,733 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:35,733 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. after waiting 0 ms 2024-12-10T12:34:35,733 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:35,733 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:35,733 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:35,735 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T12:34:35,736 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733834075736"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733834075736"}]},"ts":"1733834075736"} 2024-12-10T12:34:35,738 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-10T12:34:35,740 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T12:34:35,740 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834075740"}]},"ts":"1733834075740"} 2024-12-10T12:34:35,742 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T12:34:35,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=430faea91f05d6420a9ced51a112393c, ASSIGN}] 2024-12-10T12:34:35,802 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=430faea91f05d6420a9ced51a112393c, ASSIGN 2024-12-10T12:34:35,804 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=430faea91f05d6420a9ced51a112393c, ASSIGN; state=OFFLINE, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=false 2024-12-10T12:34:35,955 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=430faea91f05d6420a9ced51a112393c, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:34:35,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617}] 2024-12-10T12:34:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-12-10T12:34:36,112 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:36,118 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:36,118 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:34:36,119 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:36,119 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:34:36,119 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:36,119 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:36,122 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:36,124 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:34:36,124 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430faea91f05d6420a9ced51a112393c columnFamilyName A 2024-12-10T12:34:36,124 DEBUG [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:36,125 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] regionserver.HStore(327): Store=430faea91f05d6420a9ced51a112393c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:34:36,125 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:36,127 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:34:36,127 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430faea91f05d6420a9ced51a112393c columnFamilyName B 2024-12-10T12:34:36,127 DEBUG [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:36,128 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] regionserver.HStore(327): Store=430faea91f05d6420a9ced51a112393c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:34:36,128 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:36,130 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:34:36,131 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430faea91f05d6420a9ced51a112393c columnFamilyName C 2024-12-10T12:34:36,131 DEBUG [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:34:36,132 INFO [StoreOpener-430faea91f05d6420a9ced51a112393c-1 {}] regionserver.HStore(327): Store=430faea91f05d6420a9ced51a112393c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:34:36,132 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:36,133 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:36,134 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:36,136 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:34:36,138 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:36,141 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:34:36,142 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1102): Opened 430faea91f05d6420a9ced51a112393c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60467142, jitterRate=-0.09896937012672424}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:34:36,143 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:36,143 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., pid=74, masterSystemTime=1733834076111 2024-12-10T12:34:36,145 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:36,145 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:36,145 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=430faea91f05d6420a9ced51a112393c, regionState=OPEN, openSeqNum=2, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:34:36,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-12-10T12:34:36,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 in 188 msec 2024-12-10T12:34:36,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-12-10T12:34:36,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=430faea91f05d6420a9ced51a112393c, ASSIGN in 349 msec 2024-12-10T12:34:36,150 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T12:34:36,150 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834076150"}]},"ts":"1733834076150"} 2024-12-10T12:34:36,151 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T12:34:36,191 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T12:34:36,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2840 sec 2024-12-10T12:34:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-12-10T12:34:37,019 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 72 completed 2024-12-10T12:34:37,023 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7c480dfb to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@683b64c3 2024-12-10T12:34:37,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec09297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,071 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52834, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,073 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T12:34:37,075 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34856, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T12:34:37,077 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34cb3991 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e55eb7 2024-12-10T12:34:37,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dfb20f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,089 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e9ae050 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3a703d2 2024-12-10T12:34:37,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17cf7fc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,101 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2fef31f8 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14ed1e44 2024-12-10T12:34:37,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b04266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,109 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0eb04aeb to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72537a47 2024-12-10T12:34:37,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88aa519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,118 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a0e9c8f to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@36642cb 2024-12-10T12:34:37,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e998dd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,134 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c299cfb to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e4c79b8 2024-12-10T12:34:37,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a78bf6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,150 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x605827c9 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d1403c3 2024-12-10T12:34:37,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328852db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,160 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3677bd4f to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3bf0ba59 2024-12-10T12:34:37,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9e2976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,174 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x521aad6f to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6c86f707 2024-12-10T12:34:37,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e9a678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,185 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f5b2180 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34becda3 2024-12-10T12:34:37,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f7f772a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:34:37,200 DEBUG [hconnection-0x3878146f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,200 DEBUG [hconnection-0x330bd571-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,200 DEBUG [hconnection-0x189c9974-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,200 DEBUG [hconnection-0x776ab335-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:37,201 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52840, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,201 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52842, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,201 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52836, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,202 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52838, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees 2024-12-10T12:34:37,202 DEBUG [hconnection-0x3e458b6a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,203 DEBUG [hconnection-0x57865412-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-10T12:34:37,203 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52874, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,203 DEBUG [hconnection-0x10da2a7b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,203 DEBUG [hconnection-0x4bd7fb55-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,203 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52886, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,203 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:37,204 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52900, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,204 DEBUG [hconnection-0x797a5b52-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,204 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=75, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:37,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:37,204 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52904, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,205 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52916, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,207 DEBUG [hconnection-0x11535536-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:34:37,208 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52924, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:34:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:37,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:34:37,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:37,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:37,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:37,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:37,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:37,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:37,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/509b45d5a2cc4e4ba1f35e6a34809508 is 50, key is test_row_0/A:col10/1733834077208/Put/seqid=0 2024-12-10T12:34:37,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742061_1237 (size=12001) 2024-12-10T12:34:37,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/509b45d5a2cc4e4ba1f35e6a34809508 2024-12-10T12:34:37,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/b61ccbfd56154c4e8da9333eeb4b5bc3 is 50, key is test_row_0/B:col10/1733834077208/Put/seqid=0 2024-12-10T12:34:37,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834137253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834137253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834137256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834137256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834137256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742062_1238 (size=12001) 2024-12-10T12:34:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-10T12:34:37,356 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:37,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-12-10T12:34:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834137357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834137357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834137359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834137359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834137360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-10T12:34:37,509 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:37,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-12-10T12:34:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834137559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834137560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834137561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834137561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834137562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,661 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:37,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-12-10T12:34:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/b61ccbfd56154c4e8da9333eeb4b5bc3 2024-12-10T12:34:37,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/dcd497da5de2474f99d0fa38f280001a is 50, key is test_row_0/C:col10/1733834077208/Put/seqid=0 2024-12-10T12:34:37,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742063_1239 (size=12001) 2024-12-10T12:34:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-10T12:34:37,813 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:37,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-12-10T12:34:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834137861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834137861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834137863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834137864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834137864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:37,966 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:37,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-12-10T12:34:37,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:37,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:37,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:38,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/dcd497da5de2474f99d0fa38f280001a 2024-12-10T12:34:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/509b45d5a2cc4e4ba1f35e6a34809508 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/509b45d5a2cc4e4ba1f35e6a34809508 2024-12-10T12:34:38,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/509b45d5a2cc4e4ba1f35e6a34809508, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T12:34:38,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/b61ccbfd56154c4e8da9333eeb4b5bc3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b61ccbfd56154c4e8da9333eeb4b5bc3 2024-12-10T12:34:38,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b61ccbfd56154c4e8da9333eeb4b5bc3, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T12:34:38,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/dcd497da5de2474f99d0fa38f280001a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/dcd497da5de2474f99d0fa38f280001a 2024-12-10T12:34:38,118 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:38,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-12-10T12:34:38,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:38,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:38,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:38,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:38,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:38,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/dcd497da5de2474f99d0fa38f280001a, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T12:34:38,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 430faea91f05d6420a9ced51a112393c in 914ms, sequenceid=13, compaction requested=false 2024-12-10T12:34:38,123 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T12:34:38,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:38,271 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:38,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-12-10T12:34:38,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:38,271 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:34:38,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:38,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:38,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:38,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:38,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:38,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:38,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/5037aafb2da24f788ee19a872baec3ac is 50, key is test_row_0/A:col10/1733834077255/Put/seqid=0 2024-12-10T12:34:38,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742064_1240 (size=12001) 2024-12-10T12:34:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-10T12:34:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:38,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:38,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834138369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834138369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834138369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834138370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834138370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834138471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834138472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834138472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834138472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834138472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834138674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834138674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834138674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834138674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834138675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,682 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/5037aafb2da24f788ee19a872baec3ac 2024-12-10T12:34:38,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/00ffddf4b80a40e889d0c99614780ad9 is 50, key is test_row_0/B:col10/1733834077255/Put/seqid=0 2024-12-10T12:34:38,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742065_1241 (size=12001) 2024-12-10T12:34:38,703 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/00ffddf4b80a40e889d0c99614780ad9 2024-12-10T12:34:38,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/4015debc579644e48151a90a8291db02 is 50, key is test_row_0/C:col10/1733834077255/Put/seqid=0 2024-12-10T12:34:38,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742066_1242 (size=12001) 2024-12-10T12:34:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834138976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834138976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834138976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834138977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:38,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834138977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,116 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/4015debc579644e48151a90a8291db02 2024-12-10T12:34:39,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/5037aafb2da24f788ee19a872baec3ac as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/5037aafb2da24f788ee19a872baec3ac 2024-12-10T12:34:39,123 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/5037aafb2da24f788ee19a872baec3ac, entries=150, sequenceid=38, filesize=11.7 K 2024-12-10T12:34:39,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/00ffddf4b80a40e889d0c99614780ad9 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/00ffddf4b80a40e889d0c99614780ad9 2024-12-10T12:34:39,127 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/00ffddf4b80a40e889d0c99614780ad9, entries=150, sequenceid=38, filesize=11.7 K 2024-12-10T12:34:39,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/4015debc579644e48151a90a8291db02 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/4015debc579644e48151a90a8291db02 2024-12-10T12:34:39,131 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/4015debc579644e48151a90a8291db02, entries=150, sequenceid=38, filesize=11.7 K 2024-12-10T12:34:39,132 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 430faea91f05d6420a9ced51a112393c in 861ms, sequenceid=38, compaction requested=false 2024-12-10T12:34:39,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:39,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:39,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-12-10T12:34:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=76 2024-12-10T12:34:39,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-12-10T12:34:39,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9290 sec 2024-12-10T12:34:39,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees in 1.9340 sec 2024-12-10T12:34:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-10T12:34:39,306 INFO [Thread-1102 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 75 completed 2024-12-10T12:34:39,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees 2024-12-10T12:34:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-10T12:34:39,308 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:39,309 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=77, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:39,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-10T12:34:39,460 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:39,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-12-10T12:34:39,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:39,460 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-10T12:34:39,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:39,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:39,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:39,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:39,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:39,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/dec46d2d26cb4d9b8c2ea8298b629815 is 50, key is test_row_0/A:col10/1733834078369/Put/seqid=0 2024-12-10T12:34:39,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742067_1243 (size=12001) 2024-12-10T12:34:39,468 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/dec46d2d26cb4d9b8c2ea8298b629815 2024-12-10T12:34:39,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/47b079ae6f2f4f6da4859823bd8b9704 is 50, key is test_row_0/B:col10/1733834078369/Put/seqid=0 2024-12-10T12:34:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:39,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:39,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742068_1244 (size=12001) 2024-12-10T12:34:39,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834139492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834139492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834139493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834139494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834139494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834139595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834139595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834139595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834139596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834139597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-10T12:34:39,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834139798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834139798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834139799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834139799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834139800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:39,889 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/47b079ae6f2f4f6da4859823bd8b9704 2024-12-10T12:34:39,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5724ce916833435098fb7d8f94e3cbf6 is 50, key is test_row_0/C:col10/1733834078369/Put/seqid=0 2024-12-10T12:34:39,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742069_1245 (size=12001) 2024-12-10T12:34:39,902 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5724ce916833435098fb7d8f94e3cbf6 2024-12-10T12:34:39,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/dec46d2d26cb4d9b8c2ea8298b629815 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/dec46d2d26cb4d9b8c2ea8298b629815 2024-12-10T12:34:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-10T12:34:39,911 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/dec46d2d26cb4d9b8c2ea8298b629815, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T12:34:39,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/47b079ae6f2f4f6da4859823bd8b9704 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/47b079ae6f2f4f6da4859823bd8b9704 2024-12-10T12:34:39,916 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/47b079ae6f2f4f6da4859823bd8b9704, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T12:34:39,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5724ce916833435098fb7d8f94e3cbf6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5724ce916833435098fb7d8f94e3cbf6 2024-12-10T12:34:39,921 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5724ce916833435098fb7d8f94e3cbf6, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T12:34:39,922 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 430faea91f05d6420a9ced51a112393c in 462ms, sequenceid=50, compaction requested=true 2024-12-10T12:34:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-12-10T12:34:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=78 2024-12-10T12:34:39,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=78, resume processing ppid=77 2024-12-10T12:34:39,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 614 msec 2024-12-10T12:34:39,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees in 618 msec 2024-12-10T12:34:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:40,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:34:40,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:40,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:40,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:40,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:40,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:40,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:40,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/c24615754c2c409b812030030ca8ff17 is 50, key is test_row_0/A:col10/1733834080103/Put/seqid=0 2024-12-10T12:34:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834140107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834140108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834140108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834140111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834140110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742070_1246 (size=12001) 2024-12-10T12:34:40,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/c24615754c2c409b812030030ca8ff17 2024-12-10T12:34:40,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/87e88a8f50f3407690961b0b2e20a5ca is 50, key is test_row_0/B:col10/1733834080103/Put/seqid=0 2024-12-10T12:34:40,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742071_1247 (size=12001) 2024-12-10T12:34:40,152 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T12:34:40,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834140211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834140212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834140212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834140214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834140214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-10T12:34:40,411 INFO [Thread-1102 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 77 completed 2024-12-10T12:34:40,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees 2024-12-10T12:34:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-10T12:34:40,413 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:40,413 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=79, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:40,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:40,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834140413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834140414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834140414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834140415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834140417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-10T12:34:40,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/87e88a8f50f3407690961b0b2e20a5ca 2024-12-10T12:34:40,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5bcf24a5df10464895bea069d29e1534 is 50, key is test_row_0/C:col10/1733834080103/Put/seqid=0 2024-12-10T12:34:40,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742072_1248 (size=12001) 2024-12-10T12:34:40,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5bcf24a5df10464895bea069d29e1534 2024-12-10T12:34:40,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/c24615754c2c409b812030030ca8ff17 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/c24615754c2c409b812030030ca8ff17 2024-12-10T12:34:40,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/c24615754c2c409b812030030ca8ff17, entries=150, sequenceid=76, filesize=11.7 K 2024-12-10T12:34:40,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/87e88a8f50f3407690961b0b2e20a5ca as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/87e88a8f50f3407690961b0b2e20a5ca 2024-12-10T12:34:40,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/87e88a8f50f3407690961b0b2e20a5ca, entries=150, sequenceid=76, filesize=11.7 K 2024-12-10T12:34:40,564 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:40,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5bcf24a5df10464895bea069d29e1534 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5bcf24a5df10464895bea069d29e1534 2024-12-10T12:34:40,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-12-10T12:34:40,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:40,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:40,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5bcf24a5df10464895bea069d29e1534, entries=150, sequenceid=76, filesize=11.7 K 2024-12-10T12:34:40,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 430faea91f05d6420a9ced51a112393c in 467ms, sequenceid=76, compaction requested=true 2024-12-10T12:34:40,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:40,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:40,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:40,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:40,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:40,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:40,570 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:40,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:40,570 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:40,571 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:40,571 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:40,571 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/B is initiating minor compaction (all files) 2024-12-10T12:34:40,571 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/A is initiating minor compaction (all files) 2024-12-10T12:34:40,571 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/A in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:40,571 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/B in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:40,571 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/509b45d5a2cc4e4ba1f35e6a34809508, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/5037aafb2da24f788ee19a872baec3ac, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/dec46d2d26cb4d9b8c2ea8298b629815, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/c24615754c2c409b812030030ca8ff17] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=46.9 K 2024-12-10T12:34:40,571 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b61ccbfd56154c4e8da9333eeb4b5bc3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/00ffddf4b80a40e889d0c99614780ad9, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/47b079ae6f2f4f6da4859823bd8b9704, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/87e88a8f50f3407690961b0b2e20a5ca] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=46.9 K 2024-12-10T12:34:40,572 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 509b45d5a2cc4e4ba1f35e6a34809508, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733834077207 2024-12-10T12:34:40,572 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting b61ccbfd56154c4e8da9333eeb4b5bc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733834077207 2024-12-10T12:34:40,572 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5037aafb2da24f788ee19a872baec3ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733834077252 2024-12-10T12:34:40,572 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 00ffddf4b80a40e889d0c99614780ad9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733834077252 2024-12-10T12:34:40,572 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 47b079ae6f2f4f6da4859823bd8b9704, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733834078368 2024-12-10T12:34:40,572 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting dec46d2d26cb4d9b8c2ea8298b629815, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733834078368 2024-12-10T12:34:40,573 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 87e88a8f50f3407690961b0b2e20a5ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733834079493 2024-12-10T12:34:40,573 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting c24615754c2c409b812030030ca8ff17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733834079493 2024-12-10T12:34:40,581 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:40,582 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/d680d8d0cac24da7a5d5dbc6431dc235 is 50, key is test_row_0/B:col10/1733834080103/Put/seqid=0 2024-12-10T12:34:40,584 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#A#compaction#205 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:40,584 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/b69ef1e6ee304ce7b301af7fca9e9099 is 50, key is test_row_0/A:col10/1733834080103/Put/seqid=0 2024-12-10T12:34:40,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742074_1250 (size=12139) 2024-12-10T12:34:40,596 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/b69ef1e6ee304ce7b301af7fca9e9099 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/b69ef1e6ee304ce7b301af7fca9e9099 2024-12-10T12:34:40,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742073_1249 (size=12139) 2024-12-10T12:34:40,603 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 430faea91f05d6420a9ced51a112393c/A of 430faea91f05d6420a9ced51a112393c into b69ef1e6ee304ce7b301af7fca9e9099(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:40,603 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:40,603 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/A, priority=12, startTime=1733834080570; duration=0sec 2024-12-10T12:34:40,603 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:40,603 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:A 2024-12-10T12:34:40,603 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:40,604 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:40,605 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/C is initiating minor compaction (all files) 2024-12-10T12:34:40,605 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/C in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:40,605 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/dcd497da5de2474f99d0fa38f280001a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/4015debc579644e48151a90a8291db02, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5724ce916833435098fb7d8f94e3cbf6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5bcf24a5df10464895bea069d29e1534] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=46.9 K 2024-12-10T12:34:40,605 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting dcd497da5de2474f99d0fa38f280001a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733834077207 2024-12-10T12:34:40,605 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4015debc579644e48151a90a8291db02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733834077252 2024-12-10T12:34:40,606 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5724ce916833435098fb7d8f94e3cbf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733834078368 2024-12-10T12:34:40,606 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5bcf24a5df10464895bea069d29e1534, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733834079493 2024-12-10T12:34:40,614 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:40,615 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/111f28f4e6f74b718ddcafa17b44d890 is 50, key is test_row_0/C:col10/1733834080103/Put/seqid=0 2024-12-10T12:34:40,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742075_1251 (size=12139) 2024-12-10T12:34:40,630 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/111f28f4e6f74b718ddcafa17b44d890 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/111f28f4e6f74b718ddcafa17b44d890 2024-12-10T12:34:40,639 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 430faea91f05d6420a9ced51a112393c/C of 430faea91f05d6420a9ced51a112393c into 111f28f4e6f74b718ddcafa17b44d890(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:40,639 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:40,639 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/C, priority=12, startTime=1733834080570; duration=0sec 2024-12-10T12:34:40,639 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:40,640 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:C 2024-12-10T12:34:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-10T12:34:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:40,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:34:40,716 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:40,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-12-10T12:34:40,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:40,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:40,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/97727dc5b73b41a4a8d59cb6777a0280 is 50, key is test_row_0/A:col10/1733834080715/Put/seqid=0 2024-12-10T12:34:40,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834140732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834140733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834140734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834140735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834140736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742076_1252 (size=12001) 2024-12-10T12:34:40,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/97727dc5b73b41a4a8d59cb6777a0280 2024-12-10T12:34:40,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/15b1ef631b024802a03faf879605e498 is 50, key is test_row_0/B:col10/1733834080715/Put/seqid=0 2024-12-10T12:34:40,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742077_1253 (size=12001) 2024-12-10T12:34:40,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/15b1ef631b024802a03faf879605e498 2024-12-10T12:34:40,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/ed781c1740be4a7c91a41ade848b61f8 is 50, key is test_row_0/C:col10/1733834080715/Put/seqid=0 2024-12-10T12:34:40,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742078_1254 (size=12001) 2024-12-10T12:34:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834140836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834140837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834140837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834140837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834140838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:40,870 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:40,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-12-10T12:34:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:40,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:41,004 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/d680d8d0cac24da7a5d5dbc6431dc235 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d680d8d0cac24da7a5d5dbc6431dc235 2024-12-10T12:34:41,008 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 430faea91f05d6420a9ced51a112393c/B of 430faea91f05d6420a9ced51a112393c into d680d8d0cac24da7a5d5dbc6431dc235(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:41,008 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:41,008 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/B, priority=12, startTime=1733834080570; duration=0sec 2024-12-10T12:34:41,008 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:41,008 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:B 2024-12-10T12:34:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-10T12:34:41,023 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:41,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-12-10T12:34:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:41,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:41,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834141038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834141039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834141039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834141039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834141040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,175 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:41,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-12-10T12:34:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:41,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:41,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:41,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:41,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:41,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/ed781c1740be4a7c91a41ade848b61f8 2024-12-10T12:34:41,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/97727dc5b73b41a4a8d59cb6777a0280 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/97727dc5b73b41a4a8d59cb6777a0280 2024-12-10T12:34:41,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/97727dc5b73b41a4a8d59cb6777a0280, entries=150, sequenceid=89, filesize=11.7 K 2024-12-10T12:34:41,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/15b1ef631b024802a03faf879605e498 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/15b1ef631b024802a03faf879605e498 2024-12-10T12:34:41,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/15b1ef631b024802a03faf879605e498, entries=150, sequenceid=89, filesize=11.7 K 2024-12-10T12:34:41,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/ed781c1740be4a7c91a41ade848b61f8 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ed781c1740be4a7c91a41ade848b61f8 2024-12-10T12:34:41,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ed781c1740be4a7c91a41ade848b61f8, entries=150, sequenceid=89, filesize=11.7 K 2024-12-10T12:34:41,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 430faea91f05d6420a9ced51a112393c in 482ms, sequenceid=89, compaction requested=false 2024-12-10T12:34:41,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:41,328 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:41,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-12-10T12:34:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:41,328 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:34:41,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:41,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:41,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:41,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:41,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:41,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/6934578094ac41fc9e931e82ab1bccaa is 50, key is test_row_0/A:col10/1733834080732/Put/seqid=0 2024-12-10T12:34:41,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742079_1255 (size=12001) 2024-12-10T12:34:41,340 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/6934578094ac41fc9e931e82ab1bccaa 2024-12-10T12:34:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:41,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/4152948f39c748f3967099df94546351 is 50, key is test_row_0/B:col10/1733834080732/Put/seqid=0 2024-12-10T12:34:41,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834141348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834141350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834141352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834141356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834141356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742080_1256 (size=12001) 2024-12-10T12:34:41,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834141457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834141458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834141458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834141458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834141458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-10T12:34:41,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834141660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834141660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834141660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834141660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834141661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,789 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/4152948f39c748f3967099df94546351 2024-12-10T12:34:41,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/6c53644b4e8045848b5a20de3bc8bbcc is 50, key is test_row_0/C:col10/1733834080732/Put/seqid=0 2024-12-10T12:34:41,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742081_1257 (size=12001) 2024-12-10T12:34:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834141961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834141962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834141962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834141963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:41,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834141965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,212 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/6c53644b4e8045848b5a20de3bc8bbcc 2024-12-10T12:34:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/6934578094ac41fc9e931e82ab1bccaa as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/6934578094ac41fc9e931e82ab1bccaa 2024-12-10T12:34:42,219 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/6934578094ac41fc9e931e82ab1bccaa, entries=150, sequenceid=115, filesize=11.7 K 2024-12-10T12:34:42,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/4152948f39c748f3967099df94546351 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/4152948f39c748f3967099df94546351 2024-12-10T12:34:42,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/4152948f39c748f3967099df94546351, entries=150, sequenceid=115, filesize=11.7 K 2024-12-10T12:34:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/6c53644b4e8045848b5a20de3bc8bbcc as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6c53644b4e8045848b5a20de3bc8bbcc 2024-12-10T12:34:42,227 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6c53644b4e8045848b5a20de3bc8bbcc, entries=150, sequenceid=115, filesize=11.7 K 2024-12-10T12:34:42,227 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 430faea91f05d6420a9ced51a112393c in 899ms, sequenceid=115, compaction requested=true 2024-12-10T12:34:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-12-10T12:34:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=80 2024-12-10T12:34:42,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-12-10T12:34:42,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-12-10T12:34:42,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees in 1.8170 sec 2024-12-10T12:34:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:42,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:34:42,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:42,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:42,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:42,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:42,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:42,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:42,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/9b5800d74061443bad133e2b6bc91d30 is 50, key is test_row_0/A:col10/1733834082465/Put/seqid=0 2024-12-10T12:34:42,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742082_1258 (size=12051) 2024-12-10T12:34:42,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834142479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834142479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834142479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834142481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834142481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-10T12:34:42,516 INFO [Thread-1102 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 79 completed 2024-12-10T12:34:42,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees 2024-12-10T12:34:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-10T12:34:42,518 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:42,518 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:42,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834142582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834142582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834142582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834142584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834142584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-10T12:34:42,669 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:42,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-12-10T12:34:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:42,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:42,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834142785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834142785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834142785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834142787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834142787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-10T12:34:42,822 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:42,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-12-10T12:34:42,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:42,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:42,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:42,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:42,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/9b5800d74061443bad133e2b6bc91d30 2024-12-10T12:34:42,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/51e7ad22036d4ce29d160a8cd414ebea is 50, key is test_row_0/B:col10/1733834082465/Put/seqid=0 2024-12-10T12:34:42,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742083_1259 (size=12051) 2024-12-10T12:34:42,974 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:42,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-12-10T12:34:42,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:42,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:42,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:42,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:42,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834143087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834143088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834143088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834143089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834143091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-10T12:34:43,126 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:43,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-12-10T12:34:43,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:43,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,279 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:43,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-12-10T12:34:43,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:43,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/51e7ad22036d4ce29d160a8cd414ebea 2024-12-10T12:34:43,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/9f034d3e80b14003b81380d8780dfd03 is 50, key is test_row_0/C:col10/1733834082465/Put/seqid=0 2024-12-10T12:34:43,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742084_1260 (size=12051) 2024-12-10T12:34:43,431 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:43,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-12-10T12:34:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,583 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:43,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-12-10T12:34:43,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:43,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834143592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834143593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834143594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834143595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834143597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-10T12:34:43,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/9f034d3e80b14003b81380d8780dfd03 2024-12-10T12:34:43,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/9b5800d74061443bad133e2b6bc91d30 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/9b5800d74061443bad133e2b6bc91d30 2024-12-10T12:34:43,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/9b5800d74061443bad133e2b6bc91d30, entries=150, sequenceid=129, filesize=11.8 K 2024-12-10T12:34:43,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/51e7ad22036d4ce29d160a8cd414ebea as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/51e7ad22036d4ce29d160a8cd414ebea 2024-12-10T12:34:43,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/51e7ad22036d4ce29d160a8cd414ebea, entries=150, sequenceid=129, filesize=11.8 K 2024-12-10T12:34:43,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/9f034d3e80b14003b81380d8780dfd03 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9f034d3e80b14003b81380d8780dfd03 2024-12-10T12:34:43,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9f034d3e80b14003b81380d8780dfd03, entries=150, sequenceid=129, filesize=11.8 K 2024-12-10T12:34:43,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 430faea91f05d6420a9ced51a112393c in 1256ms, sequenceid=129, compaction requested=true 2024-12-10T12:34:43,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:43,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:43,722 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:43,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:43,722 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:43,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:43,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:43,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:43,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:43,723 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:43,723 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:43,724 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/A is initiating minor compaction (all files) 2024-12-10T12:34:43,724 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/B is initiating minor compaction (all files) 2024-12-10T12:34:43,724 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/A in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,724 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/B in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,724 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d680d8d0cac24da7a5d5dbc6431dc235, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/15b1ef631b024802a03faf879605e498, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/4152948f39c748f3967099df94546351, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/51e7ad22036d4ce29d160a8cd414ebea] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=47.1 K 2024-12-10T12:34:43,724 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/b69ef1e6ee304ce7b301af7fca9e9099, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/97727dc5b73b41a4a8d59cb6777a0280, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/6934578094ac41fc9e931e82ab1bccaa, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/9b5800d74061443bad133e2b6bc91d30] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=47.1 K 2024-12-10T12:34:43,724 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting d680d8d0cac24da7a5d5dbc6431dc235, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733834079493 2024-12-10T12:34:43,724 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting b69ef1e6ee304ce7b301af7fca9e9099, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733834079493 2024-12-10T12:34:43,724 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 15b1ef631b024802a03faf879605e498, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733834080106 2024-12-10T12:34:43,724 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 97727dc5b73b41a4a8d59cb6777a0280, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733834080106 2024-12-10T12:34:43,725 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 4152948f39c748f3967099df94546351, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733834080732 2024-12-10T12:34:43,725 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6934578094ac41fc9e931e82ab1bccaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733834080732 2024-12-10T12:34:43,725 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9b5800d74061443bad133e2b6bc91d30, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733834081346 2024-12-10T12:34:43,725 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 51e7ad22036d4ce29d160a8cd414ebea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733834081346 2024-12-10T12:34:43,733 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:43,733 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#A#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:43,734 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/81eb0810cf454ed2823d3b556718cfb5 is 50, key is test_row_0/A:col10/1733834082465/Put/seqid=0 2024-12-10T12:34:43,734 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/12e3ac5d27b74208afe2685344fc4e25 is 50, key is test_row_0/B:col10/1733834082465/Put/seqid=0 2024-12-10T12:34:43,735 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:43,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-12-10T12:34:43,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,735 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-10T12:34:43,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:43,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:43,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:43,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:43,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:43,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/ba43e64848a044ca843a44edffe44b5a is 50, key is test_row_0/A:col10/1733834082480/Put/seqid=0 2024-12-10T12:34:43,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742085_1261 (size=12325) 2024-12-10T12:34:43,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742086_1262 (size=12325) 2024-12-10T12:34:43,747 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/12e3ac5d27b74208afe2685344fc4e25 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/12e3ac5d27b74208afe2685344fc4e25 2024-12-10T12:34:43,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742087_1263 (size=12151) 2024-12-10T12:34:43,752 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 430faea91f05d6420a9ced51a112393c/B of 430faea91f05d6420a9ced51a112393c into 12e3ac5d27b74208afe2685344fc4e25(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:43,752 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:43,752 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/B, priority=12, startTime=1733834083722; duration=0sec 2024-12-10T12:34:43,752 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:43,752 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:B 2024-12-10T12:34:43,752 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:43,753 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:43,753 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/C is initiating minor compaction (all files) 2024-12-10T12:34:43,753 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/C in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:43,753 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/111f28f4e6f74b718ddcafa17b44d890, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ed781c1740be4a7c91a41ade848b61f8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6c53644b4e8045848b5a20de3bc8bbcc, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9f034d3e80b14003b81380d8780dfd03] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=47.1 K 2024-12-10T12:34:43,754 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 111f28f4e6f74b718ddcafa17b44d890, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733834079493 2024-12-10T12:34:43,754 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ed781c1740be4a7c91a41ade848b61f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733834080106 2024-12-10T12:34:43,754 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 6c53644b4e8045848b5a20de3bc8bbcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733834080732 2024-12-10T12:34:43,755 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 9f034d3e80b14003b81380d8780dfd03, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733834081346 2024-12-10T12:34:43,761 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:43,762 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/48adb9e95925475a912aa1a6bce659bf is 50, key is test_row_0/C:col10/1733834082465/Put/seqid=0 2024-12-10T12:34:43,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742088_1264 (size=12325) 2024-12-10T12:34:43,769 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/48adb9e95925475a912aa1a6bce659bf as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/48adb9e95925475a912aa1a6bce659bf 2024-12-10T12:34:43,773 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 430faea91f05d6420a9ced51a112393c/C of 430faea91f05d6420a9ced51a112393c into 48adb9e95925475a912aa1a6bce659bf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:43,773 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:43,773 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/C, priority=12, startTime=1733834083723; duration=0sec 2024-12-10T12:34:43,773 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:43,773 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:C 2024-12-10T12:34:44,148 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/81eb0810cf454ed2823d3b556718cfb5 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/81eb0810cf454ed2823d3b556718cfb5 2024-12-10T12:34:44,149 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/ba43e64848a044ca843a44edffe44b5a 2024-12-10T12:34:44,153 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 430faea91f05d6420a9ced51a112393c/A of 430faea91f05d6420a9ced51a112393c into 81eb0810cf454ed2823d3b556718cfb5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:44,153 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:44,153 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/A, priority=12, startTime=1733834083722; duration=0sec 2024-12-10T12:34:44,153 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:44,153 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:A 2024-12-10T12:34:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/3fbff05924104de4bf47a56d8e755054 is 50, key is test_row_0/B:col10/1733834082480/Put/seqid=0 2024-12-10T12:34:44,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742089_1265 (size=12151) 2024-12-10T12:34:44,560 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/3fbff05924104de4bf47a56d8e755054 2024-12-10T12:34:44,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/68a55516fa30475697e55b997160242c is 50, key is test_row_0/C:col10/1733834082480/Put/seqid=0 2024-12-10T12:34:44,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742090_1266 (size=12151) 2024-12-10T12:34:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:44,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-10T12:34:44,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834144601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834144601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834144626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834144626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834144626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834144727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834144727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834144728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834144728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834144729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834144930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834144930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834144930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834144931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834144931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:44,998 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/68a55516fa30475697e55b997160242c 2024-12-10T12:34:45,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/ba43e64848a044ca843a44edffe44b5a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ba43e64848a044ca843a44edffe44b5a 2024-12-10T12:34:45,005 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ba43e64848a044ca843a44edffe44b5a, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T12:34:45,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/3fbff05924104de4bf47a56d8e755054 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3fbff05924104de4bf47a56d8e755054 2024-12-10T12:34:45,009 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3fbff05924104de4bf47a56d8e755054, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T12:34:45,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/68a55516fa30475697e55b997160242c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/68a55516fa30475697e55b997160242c 2024-12-10T12:34:45,014 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/68a55516fa30475697e55b997160242c, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T12:34:45,015 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 430faea91f05d6420a9ced51a112393c in 1279ms, sequenceid=151, compaction requested=false 2024-12-10T12:34:45,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:45,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:45,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-12-10T12:34:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=82 2024-12-10T12:34:45,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-12-10T12:34:45,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4980 sec 2024-12-10T12:34:45,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees in 2.5010 sec 2024-12-10T12:34:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:45,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-10T12:34:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:45,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/159b66813cc34d3b918600f47a542659 is 50, key is test_row_0/A:col10/1733834085232/Put/seqid=0 2024-12-10T12:34:45,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742091_1267 (size=12151) 2024-12-10T12:34:45,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834145243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834145246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834145247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834145247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834145247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834145348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834145350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834145351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834145351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834145351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834145550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834145553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834145553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834145554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834145554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/159b66813cc34d3b918600f47a542659 2024-12-10T12:34:45,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/1f6549529161485fbd6e76046cf9b6a8 is 50, key is test_row_0/B:col10/1733834085232/Put/seqid=0 2024-12-10T12:34:45,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742092_1268 (size=12151) 2024-12-10T12:34:45,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834145853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834145855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834145856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834145856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834145857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/1f6549529161485fbd6e76046cf9b6a8 2024-12-10T12:34:46,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/f9e8725fd8af4301b713565ffc59cf79 is 50, key is test_row_0/C:col10/1733834085232/Put/seqid=0 2024-12-10T12:34:46,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742093_1269 (size=12151) 2024-12-10T12:34:46,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/f9e8725fd8af4301b713565ffc59cf79 2024-12-10T12:34:46,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/159b66813cc34d3b918600f47a542659 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/159b66813cc34d3b918600f47a542659 2024-12-10T12:34:46,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/159b66813cc34d3b918600f47a542659, entries=150, sequenceid=169, filesize=11.9 K 2024-12-10T12:34:46,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/1f6549529161485fbd6e76046cf9b6a8 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1f6549529161485fbd6e76046cf9b6a8 2024-12-10T12:34:46,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1f6549529161485fbd6e76046cf9b6a8, entries=150, sequenceid=169, filesize=11.9 K 2024-12-10T12:34:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/f9e8725fd8af4301b713565ffc59cf79 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/f9e8725fd8af4301b713565ffc59cf79 2024-12-10T12:34:46,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/f9e8725fd8af4301b713565ffc59cf79, entries=150, sequenceid=169, filesize=11.9 K 2024-12-10T12:34:46,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 430faea91f05d6420a9ced51a112393c in 858ms, sequenceid=169, compaction requested=true 2024-12-10T12:34:46,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:46,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:46,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:46,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:46,092 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:46,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:46,092 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:46,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:46,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:46,093 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:46,093 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:46,093 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/B is initiating minor compaction (all files) 2024-12-10T12:34:46,093 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/A is initiating minor compaction (all files) 2024-12-10T12:34:46,093 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/B in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:46,093 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/A in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:46,093 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/12e3ac5d27b74208afe2685344fc4e25, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3fbff05924104de4bf47a56d8e755054, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1f6549529161485fbd6e76046cf9b6a8] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=35.8 K 2024-12-10T12:34:46,093 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/81eb0810cf454ed2823d3b556718cfb5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ba43e64848a044ca843a44edffe44b5a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/159b66813cc34d3b918600f47a542659] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=35.8 K 2024-12-10T12:34:46,093 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 12e3ac5d27b74208afe2685344fc4e25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733834081346 2024-12-10T12:34:46,093 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 81eb0810cf454ed2823d3b556718cfb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733834081346 2024-12-10T12:34:46,094 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 3fbff05924104de4bf47a56d8e755054, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733834082478 2024-12-10T12:34:46,094 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting ba43e64848a044ca843a44edffe44b5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733834082478 2024-12-10T12:34:46,094 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1f6549529161485fbd6e76046cf9b6a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733834084602 2024-12-10T12:34:46,094 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 159b66813cc34d3b918600f47a542659, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733834084602 2024-12-10T12:34:46,100 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:46,100 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:46,101 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/f9f4ed66bf1942bfa4c376401f8786a0 is 50, key is test_row_0/B:col10/1733834085232/Put/seqid=0 2024-12-10T12:34:46,101 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/d00817b55b064346ab2b4365b3489541 is 50, key is test_row_0/A:col10/1733834085232/Put/seqid=0 2024-12-10T12:34:46,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742094_1270 (size=12527) 2024-12-10T12:34:46,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742095_1271 (size=12527) 2024-12-10T12:34:46,109 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/f9f4ed66bf1942bfa4c376401f8786a0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/f9f4ed66bf1942bfa4c376401f8786a0 2024-12-10T12:34:46,112 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/d00817b55b064346ab2b4365b3489541 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/d00817b55b064346ab2b4365b3489541 2024-12-10T12:34:46,115 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/B of 430faea91f05d6420a9ced51a112393c into f9f4ed66bf1942bfa4c376401f8786a0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:46,115 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:46,115 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/B, priority=13, startTime=1733834086092; duration=0sec 2024-12-10T12:34:46,115 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:46,115 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:B 2024-12-10T12:34:46,115 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:46,116 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:46,116 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/C is initiating minor compaction (all files) 2024-12-10T12:34:46,116 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/C in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:46,116 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/48adb9e95925475a912aa1a6bce659bf, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/68a55516fa30475697e55b997160242c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/f9e8725fd8af4301b713565ffc59cf79] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=35.8 K 2024-12-10T12:34:46,117 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 48adb9e95925475a912aa1a6bce659bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733834081346 2024-12-10T12:34:46,117 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 68a55516fa30475697e55b997160242c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733834082478 2024-12-10T12:34:46,117 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting f9e8725fd8af4301b713565ffc59cf79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733834084602 2024-12-10T12:34:46,118 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/A of 430faea91f05d6420a9ced51a112393c into d00817b55b064346ab2b4365b3489541(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:46,118 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:46,118 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/A, priority=13, startTime=1733834086092; duration=0sec 2024-12-10T12:34:46,118 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:46,118 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:A 2024-12-10T12:34:46,124 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#C#compaction#227 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:46,125 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/9a9861b2ce134d01a8bbfc60e6d746d0 is 50, key is test_row_0/C:col10/1733834085232/Put/seqid=0 2024-12-10T12:34:46,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742096_1272 (size=12527) 2024-12-10T12:34:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:46,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-10T12:34:46,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:46,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:46,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:46,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:46,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:46,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:46,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/84eb3a15e78f4aacb0221e23afd53592 is 50, key is test_row_0/A:col10/1733834086358/Put/seqid=0 2024-12-10T12:34:46,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834146364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834146365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834146365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834146365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834146366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742097_1273 (size=14541) 2024-12-10T12:34:46,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834146467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834146468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834146468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834146468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834146468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,549 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/9a9861b2ce134d01a8bbfc60e6d746d0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9a9861b2ce134d01a8bbfc60e6d746d0 2024-12-10T12:34:46,553 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/C of 430faea91f05d6420a9ced51a112393c into 9a9861b2ce134d01a8bbfc60e6d746d0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:46,553 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:46,553 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/C, priority=13, startTime=1733834086092; duration=0sec 2024-12-10T12:34:46,554 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:46,554 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:C 2024-12-10T12:34:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-10T12:34:46,622 INFO [Thread-1102 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 81 completed 2024-12-10T12:34:46,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2024-12-10T12:34:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-10T12:34:46,625 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:46,625 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:46,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:46,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834146669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834146669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834146670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834146670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834146671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-10T12:34:46,776 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:46,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T12:34:46,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:46,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:46,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:46,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:46,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:46,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/84eb3a15e78f4aacb0221e23afd53592 2024-12-10T12:34:46,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/af5da414cade4290956279ebe7df8703 is 50, key is test_row_0/B:col10/1733834086358/Put/seqid=0 2024-12-10T12:34:46,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742098_1274 (size=12151) 2024-12-10T12:34:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-10T12:34:46,929 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:46,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T12:34:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:46,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:46,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:46,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834146972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834146972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834146973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834146973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:46,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834146973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:47,081 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:47,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T12:34:47,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:47,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/af5da414cade4290956279ebe7df8703 2024-12-10T12:34:47,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/95cfe91dd5444033a4a0c96770e5ef0a is 50, key is test_row_0/C:col10/1733834086358/Put/seqid=0 2024-12-10T12:34:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-10T12:34:47,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742099_1275 (size=12151) 2024-12-10T12:34:47,233 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:47,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T12:34:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:47,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,385 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:47,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T12:34:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:47,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834147475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:47,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834147475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:47,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834147476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:47,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834147478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:47,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834147478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:47,538 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:47,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T12:34:47,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:47,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:47,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:47,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:47,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/95cfe91dd5444033a4a0c96770e5ef0a 2024-12-10T12:34:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/84eb3a15e78f4aacb0221e23afd53592 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/84eb3a15e78f4aacb0221e23afd53592 2024-12-10T12:34:47,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/84eb3a15e78f4aacb0221e23afd53592, entries=200, sequenceid=195, filesize=14.2 K 2024-12-10T12:34:47,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/af5da414cade4290956279ebe7df8703 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/af5da414cade4290956279ebe7df8703 2024-12-10T12:34:47,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/af5da414cade4290956279ebe7df8703, entries=150, sequenceid=195, filesize=11.9 K 2024-12-10T12:34:47,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/95cfe91dd5444033a4a0c96770e5ef0a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/95cfe91dd5444033a4a0c96770e5ef0a 2024-12-10T12:34:47,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/95cfe91dd5444033a4a0c96770e5ef0a, entries=150, sequenceid=195, filesize=11.9 K 2024-12-10T12:34:47,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 430faea91f05d6420a9ced51a112393c in 1288ms, sequenceid=195, compaction requested=false 2024-12-10T12:34:47,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:47,690 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:47,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-12-10T12:34:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:47,691 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:34:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:47,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/260261a75d784c0aa99352d78eb6aa5d is 50, key is test_row_0/A:col10/1733834086365/Put/seqid=0 2024-12-10T12:34:47,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742100_1276 (size=12151) 2024-12-10T12:34:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-10T12:34:48,111 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/260261a75d784c0aa99352d78eb6aa5d 2024-12-10T12:34:48,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/ef5f2a52d3a04074a787bae12c4d530b is 50, key is test_row_0/B:col10/1733834086365/Put/seqid=0 2024-12-10T12:34:48,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742101_1277 (size=12151) 2024-12-10T12:34:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:48,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:48,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834148492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834148493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834148493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834148494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834148494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/ef5f2a52d3a04074a787bae12c4d530b 2024-12-10T12:34:48,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/26cc1160959b48e58c1e035cd8e81881 is 50, key is test_row_0/C:col10/1733834086365/Put/seqid=0 2024-12-10T12:34:48,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742102_1278 (size=12151) 2024-12-10T12:34:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834148595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834148596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834148596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834148596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834148596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-10T12:34:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834148798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834148798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834148798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834148799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834148799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:48,932 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/26cc1160959b48e58c1e035cd8e81881 2024-12-10T12:34:48,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/260261a75d784c0aa99352d78eb6aa5d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/260261a75d784c0aa99352d78eb6aa5d 2024-12-10T12:34:48,939 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/260261a75d784c0aa99352d78eb6aa5d, entries=150, sequenceid=209, filesize=11.9 K 2024-12-10T12:34:48,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/ef5f2a52d3a04074a787bae12c4d530b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ef5f2a52d3a04074a787bae12c4d530b 2024-12-10T12:34:48,943 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ef5f2a52d3a04074a787bae12c4d530b, entries=150, sequenceid=209, filesize=11.9 K 2024-12-10T12:34:48,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/26cc1160959b48e58c1e035cd8e81881 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/26cc1160959b48e58c1e035cd8e81881 2024-12-10T12:34:48,947 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/26cc1160959b48e58c1e035cd8e81881, entries=150, sequenceid=209, filesize=11.9 K 2024-12-10T12:34:48,947 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 430faea91f05d6420a9ced51a112393c in 1256ms, sequenceid=209, compaction requested=true 2024-12-10T12:34:48,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:48,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-12-10T12:34:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=84 2024-12-10T12:34:48,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=84, resume processing ppid=83 2024-12-10T12:34:48,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3230 sec 2024-12-10T12:34:48,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 2.3270 sec 2024-12-10T12:34:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:49,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:34:49,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:49,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:49,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:49,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:49,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:49,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:49,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/500b446e9ed6452fadcc9751b846e3ec is 50, key is test_row_0/A:col10/1733834088492/Put/seqid=0 2024-12-10T12:34:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834149106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742103_1279 (size=14541) 2024-12-10T12:34:49,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834149107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834149107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834149107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834149108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834149209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834149211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834149211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834149211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834149211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834149410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834149413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834149413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834149413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834149414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/500b446e9ed6452fadcc9751b846e3ec 2024-12-10T12:34:49,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/94d5d8d87f2f4bc5b76480887aec80d1 is 50, key is test_row_0/B:col10/1733834088492/Put/seqid=0 2024-12-10T12:34:49,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742104_1280 (size=12151) 2024-12-10T12:34:49,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834149715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834149717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834149717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834149717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834149718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:49,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/94d5d8d87f2f4bc5b76480887aec80d1 2024-12-10T12:34:49,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5861437e8aae4747832b39e05ba3026b is 50, key is test_row_0/C:col10/1733834088492/Put/seqid=0 2024-12-10T12:34:49,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742105_1281 (size=12151) 2024-12-10T12:34:50,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834150218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:50,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834150221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:50,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834150222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:50,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834150223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:50,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834150223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:50,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5861437e8aae4747832b39e05ba3026b 2024-12-10T12:34:50,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/500b446e9ed6452fadcc9751b846e3ec as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/500b446e9ed6452fadcc9751b846e3ec 2024-12-10T12:34:50,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/500b446e9ed6452fadcc9751b846e3ec, entries=200, sequenceid=234, filesize=14.2 K 2024-12-10T12:34:50,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/94d5d8d87f2f4bc5b76480887aec80d1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/94d5d8d87f2f4bc5b76480887aec80d1 2024-12-10T12:34:50,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/94d5d8d87f2f4bc5b76480887aec80d1, entries=150, sequenceid=234, filesize=11.9 K 2024-12-10T12:34:50,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5861437e8aae4747832b39e05ba3026b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5861437e8aae4747832b39e05ba3026b 2024-12-10T12:34:50,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5861437e8aae4747832b39e05ba3026b, entries=150, sequenceid=234, filesize=11.9 K 2024-12-10T12:34:50,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 430faea91f05d6420a9ced51a112393c in 1274ms, sequenceid=234, compaction requested=true 2024-12-10T12:34:50,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:50,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:50,376 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:50,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:50,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:50,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:50,376 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:50,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:50,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:50,377 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:50,377 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/A is initiating minor compaction (all files) 2024-12-10T12:34:50,377 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/A in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:50,378 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/d00817b55b064346ab2b4365b3489541, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/84eb3a15e78f4aacb0221e23afd53592, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/260261a75d784c0aa99352d78eb6aa5d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/500b446e9ed6452fadcc9751b846e3ec] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=52.5 K 2024-12-10T12:34:50,378 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:50,378 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/B is initiating minor compaction (all files) 2024-12-10T12:34:50,378 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting d00817b55b064346ab2b4365b3489541, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733834084602 2024-12-10T12:34:50,378 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/B in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:50,378 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/f9f4ed66bf1942bfa4c376401f8786a0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/af5da414cade4290956279ebe7df8703, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ef5f2a52d3a04074a787bae12c4d530b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/94d5d8d87f2f4bc5b76480887aec80d1] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=47.8 K 2024-12-10T12:34:50,378 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 84eb3a15e78f4aacb0221e23afd53592, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733834085242 2024-12-10T12:34:50,378 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting f9f4ed66bf1942bfa4c376401f8786a0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733834084602 2024-12-10T12:34:50,378 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 260261a75d784c0aa99352d78eb6aa5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733834086363 2024-12-10T12:34:50,378 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting af5da414cade4290956279ebe7df8703, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733834085247 2024-12-10T12:34:50,379 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ef5f2a52d3a04074a787bae12c4d530b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733834086363 2024-12-10T12:34:50,379 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 500b446e9ed6452fadcc9751b846e3ec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733834088492 2024-12-10T12:34:50,379 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 94d5d8d87f2f4bc5b76480887aec80d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733834088492 2024-12-10T12:34:50,386 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#B#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:50,386 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/3e445025cd0c434b9b0243ef77961ef6 is 50, key is test_row_0/B:col10/1733834088492/Put/seqid=0 2024-12-10T12:34:50,387 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#A#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:50,387 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/2e9dfff50d6d498ca24aabb1a5668eed is 50, key is test_row_0/A:col10/1733834088492/Put/seqid=0 2024-12-10T12:34:50,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742106_1282 (size=12663) 2024-12-10T12:34:50,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742107_1283 (size=12663) 2024-12-10T12:34:50,397 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/3e445025cd0c434b9b0243ef77961ef6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3e445025cd0c434b9b0243ef77961ef6 2024-12-10T12:34:50,403 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 430faea91f05d6420a9ced51a112393c/B of 430faea91f05d6420a9ced51a112393c into 3e445025cd0c434b9b0243ef77961ef6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:50,403 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:50,403 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/B, priority=12, startTime=1733834090376; duration=0sec 2024-12-10T12:34:50,403 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:50,403 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:B 2024-12-10T12:34:50,404 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:34:50,405 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:34:50,405 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/C is initiating minor compaction (all files) 2024-12-10T12:34:50,405 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/C in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:50,405 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9a9861b2ce134d01a8bbfc60e6d746d0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/95cfe91dd5444033a4a0c96770e5ef0a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/26cc1160959b48e58c1e035cd8e81881, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5861437e8aae4747832b39e05ba3026b] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=47.8 K 2024-12-10T12:34:50,406 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 9a9861b2ce134d01a8bbfc60e6d746d0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733834084602 2024-12-10T12:34:50,406 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 95cfe91dd5444033a4a0c96770e5ef0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733834085247 2024-12-10T12:34:50,407 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 26cc1160959b48e58c1e035cd8e81881, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733834086363 2024-12-10T12:34:50,407 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5861437e8aae4747832b39e05ba3026b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733834088492 2024-12-10T12:34:50,415 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:50,415 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/c71efb11b4514ac4bcdeafb4a99e6ca8 is 50, key is test_row_0/C:col10/1733834088492/Put/seqid=0 2024-12-10T12:34:50,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742108_1284 (size=12663) 2024-12-10T12:34:50,440 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/c71efb11b4514ac4bcdeafb4a99e6ca8 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/c71efb11b4514ac4bcdeafb4a99e6ca8 2024-12-10T12:34:50,445 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 430faea91f05d6420a9ced51a112393c/C of 430faea91f05d6420a9ced51a112393c into c71efb11b4514ac4bcdeafb4a99e6ca8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:50,445 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:50,445 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/C, priority=12, startTime=1733834090376; duration=0sec 2024-12-10T12:34:50,445 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:50,445 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:C 2024-12-10T12:34:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-10T12:34:50,728 INFO [Thread-1102 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 83 completed 2024-12-10T12:34:50,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-12-10T12:34:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-10T12:34:50,730 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:50,731 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:50,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:50,795 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/2e9dfff50d6d498ca24aabb1a5668eed as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/2e9dfff50d6d498ca24aabb1a5668eed 2024-12-10T12:34:50,799 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 430faea91f05d6420a9ced51a112393c/A of 430faea91f05d6420a9ced51a112393c into 2e9dfff50d6d498ca24aabb1a5668eed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:50,799 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:50,799 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/A, priority=12, startTime=1733834090376; duration=0sec 2024-12-10T12:34:50,799 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:50,799 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:A 2024-12-10T12:34:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-10T12:34:50,882 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:50,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-10T12:34:50,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:50,883 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-10T12:34:50,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:50,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:50,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:50,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:50,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:50,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/e3aaf5507f3345bfb9605007901d14b0 is 50, key is test_row_0/A:col10/1733834089105/Put/seqid=0 2024-12-10T12:34:50,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742109_1285 (size=12151) 2024-12-10T12:34:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-10T12:34:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:51,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834151237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834151237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834151238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834151238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834151239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,290 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/e3aaf5507f3345bfb9605007901d14b0 2024-12-10T12:34:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/218f161a0191437da3598386ee9057a1 is 50, key is test_row_0/B:col10/1733834089105/Put/seqid=0 2024-12-10T12:34:51,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742110_1286 (size=12151) 2024-12-10T12:34:51,300 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/218f161a0191437da3598386ee9057a1 2024-12-10T12:34:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/6b2993b91fde4624876859bfa40476cd is 50, key is test_row_0/C:col10/1733834089105/Put/seqid=0 2024-12-10T12:34:51,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742111_1287 (size=12151) 2024-12-10T12:34:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-10T12:34:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834151340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834151340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834151340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834151340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834151341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834151543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834151542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834151543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834151543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834151544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,713 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/6b2993b91fde4624876859bfa40476cd 2024-12-10T12:34:51,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/e3aaf5507f3345bfb9605007901d14b0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/e3aaf5507f3345bfb9605007901d14b0 2024-12-10T12:34:51,720 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/e3aaf5507f3345bfb9605007901d14b0, entries=150, sequenceid=249, filesize=11.9 K 2024-12-10T12:34:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/218f161a0191437da3598386ee9057a1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/218f161a0191437da3598386ee9057a1 2024-12-10T12:34:51,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/218f161a0191437da3598386ee9057a1, entries=150, sequenceid=249, filesize=11.9 K 2024-12-10T12:34:51,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/6b2993b91fde4624876859bfa40476cd as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6b2993b91fde4624876859bfa40476cd 2024-12-10T12:34:51,746 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6b2993b91fde4624876859bfa40476cd, entries=150, sequenceid=249, filesize=11.9 K 2024-12-10T12:34:51,746 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 430faea91f05d6420a9ced51a112393c in 863ms, sequenceid=249, compaction requested=false 2024-12-10T12:34:51,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:51,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:51,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-10T12:34:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=86 2024-12-10T12:34:51,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-12-10T12:34:51,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-12-10T12:34:51,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 1.0200 sec 2024-12-10T12:34:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-10T12:34:51,833 INFO [Thread-1102 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 85 completed 2024-12-10T12:34:51,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-12-10T12:34:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-10T12:34:51,835 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:51,835 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:51,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:51,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:34:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:51,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:51,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/16f5fea2be1748d58e199afa107ccadb is 50, key is test_row_0/A:col10/1733834091238/Put/seqid=0 2024-12-10T12:34:51,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834151850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834151850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834151851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742112_1288 (size=12301) 2024-12-10T12:34:51,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834151853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834151853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/16f5fea2be1748d58e199afa107ccadb 2024-12-10T12:34:51,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/ff56ccfed9124eb49df27ba921ffa242 is 50, key is test_row_0/B:col10/1733834091238/Put/seqid=0 2024-12-10T12:34:51,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742113_1289 (size=12301) 2024-12-10T12:34:51,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/ff56ccfed9124eb49df27ba921ffa242 2024-12-10T12:34:51,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/e74c990e2d8f46d6a5cfd046b1b8a8e0 is 50, key is test_row_0/C:col10/1733834091238/Put/seqid=0 2024-12-10T12:34:51,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742114_1290 (size=12301) 2024-12-10T12:34:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-10T12:34:51,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834151954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834151954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834151954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834151956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834151956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:51,986 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:51,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T12:34:51,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:51,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:51,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:51,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:51,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-10T12:34:52,139 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:52,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T12:34:52,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:52,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:52,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:52,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:52,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:52,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834152155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834152156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834152156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834152158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834152158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/e74c990e2d8f46d6a5cfd046b1b8a8e0 2024-12-10T12:34:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/16f5fea2be1748d58e199afa107ccadb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/16f5fea2be1748d58e199afa107ccadb 2024-12-10T12:34:52,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/16f5fea2be1748d58e199afa107ccadb, entries=150, sequenceid=275, filesize=12.0 K 2024-12-10T12:34:52,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/ff56ccfed9124eb49df27ba921ffa242 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ff56ccfed9124eb49df27ba921ffa242 2024-12-10T12:34:52,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ff56ccfed9124eb49df27ba921ffa242, entries=150, sequenceid=275, filesize=12.0 K 2024-12-10T12:34:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/e74c990e2d8f46d6a5cfd046b1b8a8e0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e74c990e2d8f46d6a5cfd046b1b8a8e0 2024-12-10T12:34:52,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e74c990e2d8f46d6a5cfd046b1b8a8e0, entries=150, sequenceid=275, filesize=12.0 K 2024-12-10T12:34:52,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 430faea91f05d6420a9ced51a112393c in 443ms, sequenceid=275, compaction requested=true 2024-12-10T12:34:52,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:52,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:52,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:52,290 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:52,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:52,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:52,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:52,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T12:34:52,290 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:52,291 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:52,291 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:52,291 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/A is initiating minor compaction (all files) 2024-12-10T12:34:52,291 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/B is initiating minor compaction (all files) 2024-12-10T12:34:52,291 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/A in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:52,291 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/B in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:52,291 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/2e9dfff50d6d498ca24aabb1a5668eed, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/e3aaf5507f3345bfb9605007901d14b0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/16f5fea2be1748d58e199afa107ccadb] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=36.2 K 2024-12-10T12:34:52,291 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3e445025cd0c434b9b0243ef77961ef6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/218f161a0191437da3598386ee9057a1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ff56ccfed9124eb49df27ba921ffa242] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=36.2 K 2024-12-10T12:34:52,291 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:52,291 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2e9dfff50d6d498ca24aabb1a5668eed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733834088492 2024-12-10T12:34:52,291 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 3e445025cd0c434b9b0243ef77961ef6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733834088492 2024-12-10T12:34:52,292 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting e3aaf5507f3345bfb9605007901d14b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733834089105 2024-12-10T12:34:52,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-10T12:34:52,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:52,292 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 16f5fea2be1748d58e199afa107ccadb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733834091237 2024-12-10T12:34:52,292 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-10T12:34:52,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:52,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:52,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:52,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:52,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:52,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:52,293 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 218f161a0191437da3598386ee9057a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733834089105 2024-12-10T12:34:52,293 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ff56ccfed9124eb49df27ba921ffa242, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733834091237 2024-12-10T12:34:52,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/4ce7f090a1ae4a51a52388196e01f956 is 50, key is test_row_0/A:col10/1733834091852/Put/seqid=0 2024-12-10T12:34:52,302 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:52,303 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/ef340ea21fe649c6a8b8270d6cb91ae4 is 50, key is test_row_0/A:col10/1733834091238/Put/seqid=0 2024-12-10T12:34:52,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742116_1292 (size=12915) 2024-12-10T12:34:52,307 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#B#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:52,307 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/df67994a5d97487f833c844dc1a2cfdd is 50, key is test_row_0/B:col10/1733834091238/Put/seqid=0 2024-12-10T12:34:52,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742115_1291 (size=12301) 2024-12-10T12:34:52,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/4ce7f090a1ae4a51a52388196e01f956 2024-12-10T12:34:52,314 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/ef340ea21fe649c6a8b8270d6cb91ae4 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ef340ea21fe649c6a8b8270d6cb91ae4 2024-12-10T12:34:52,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742117_1293 (size=12915) 2024-12-10T12:34:52,321 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/A of 430faea91f05d6420a9ced51a112393c into ef340ea21fe649c6a8b8270d6cb91ae4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:52,321 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:52,321 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/A, priority=13, startTime=1733834092290; duration=0sec 2024-12-10T12:34:52,321 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:52,321 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:A 2024-12-10T12:34:52,321 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:52,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/903cc2f0bea14bc0872f52022457b4c0 is 50, key is test_row_0/B:col10/1733834091852/Put/seqid=0 2024-12-10T12:34:52,323 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/df67994a5d97487f833c844dc1a2cfdd as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/df67994a5d97487f833c844dc1a2cfdd 2024-12-10T12:34:52,323 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:52,323 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/C is initiating minor compaction (all files) 2024-12-10T12:34:52,324 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/C in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:52,324 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/c71efb11b4514ac4bcdeafb4a99e6ca8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6b2993b91fde4624876859bfa40476cd, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e74c990e2d8f46d6a5cfd046b1b8a8e0] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=36.2 K 2024-12-10T12:34:52,324 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting c71efb11b4514ac4bcdeafb4a99e6ca8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733834088492 2024-12-10T12:34:52,324 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6b2993b91fde4624876859bfa40476cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733834089105 2024-12-10T12:34:52,325 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting e74c990e2d8f46d6a5cfd046b1b8a8e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733834091237 2024-12-10T12:34:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742118_1294 (size=12301) 2024-12-10T12:34:52,328 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/B of 430faea91f05d6420a9ced51a112393c into df67994a5d97487f833c844dc1a2cfdd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:52,328 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:52,328 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/903cc2f0bea14bc0872f52022457b4c0 2024-12-10T12:34:52,328 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/B, priority=13, startTime=1733834092290; duration=0sec 2024-12-10T12:34:52,328 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:52,328 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:B 2024-12-10T12:34:52,332 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#C#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:52,332 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/73c0040c8c7f4c8c8af424b9d52c0767 is 50, key is test_row_0/C:col10/1733834091238/Put/seqid=0 2024-12-10T12:34:52,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/117f70b5a38e43a79b769c9424d1be35 is 50, key is test_row_0/C:col10/1733834091852/Put/seqid=0 2024-12-10T12:34:52,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742119_1295 (size=12915) 2024-12-10T12:34:52,357 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/73c0040c8c7f4c8c8af424b9d52c0767 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/73c0040c8c7f4c8c8af424b9d52c0767 2024-12-10T12:34:52,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742120_1296 (size=12301) 2024-12-10T12:34:52,361 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/C of 430faea91f05d6420a9ced51a112393c into 73c0040c8c7f4c8c8af424b9d52c0767(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:52,362 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:52,362 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/C, priority=13, startTime=1733834092290; duration=0sec 2024-12-10T12:34:52,362 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:52,362 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:C 2024-12-10T12:34:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-10T12:34:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:52,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:52,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834152474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834152475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834152477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834152478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834152478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834152578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834152579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834152581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834152582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834152582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,758 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/117f70b5a38e43a79b769c9424d1be35 2024-12-10T12:34:52,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/4ce7f090a1ae4a51a52388196e01f956 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/4ce7f090a1ae4a51a52388196e01f956 2024-12-10T12:34:52,765 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/4ce7f090a1ae4a51a52388196e01f956, entries=150, sequenceid=285, filesize=12.0 K 2024-12-10T12:34:52,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/903cc2f0bea14bc0872f52022457b4c0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/903cc2f0bea14bc0872f52022457b4c0 2024-12-10T12:34:52,769 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/903cc2f0bea14bc0872f52022457b4c0, entries=150, sequenceid=285, filesize=12.0 K 2024-12-10T12:34:52,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/117f70b5a38e43a79b769c9424d1be35 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/117f70b5a38e43a79b769c9424d1be35 2024-12-10T12:34:52,773 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/117f70b5a38e43a79b769c9424d1be35, entries=150, sequenceid=285, filesize=12.0 K 2024-12-10T12:34:52,773 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3040): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=167.72 KB/171750 for 430faea91f05d6420a9ced51a112393c in 481ms, sequenceid=285, compaction requested=false 2024-12-10T12:34:52,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:52,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:52,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-10T12:34:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=88 2024-12-10T12:34:52,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-12-10T12:34:52,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 939 msec 2024-12-10T12:34:52,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 942 msec 2024-12-10T12:34:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:52,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-12-10T12:34:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:52,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/bf81751157614ce793d94ffa2f4f08d6 is 50, key is test_row_0/A:col10/1733834092476/Put/seqid=0 2024-12-10T12:34:52,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834152785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834152786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834152786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834152788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834152788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742121_1297 (size=14741) 2024-12-10T12:34:52,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834152889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834152889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834152889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834152890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834152890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-10T12:34:52,937 INFO [Thread-1102 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 87 completed 2024-12-10T12:34:52,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-10T12:34:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-10T12:34:52,939 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:52,939 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:52,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-10T12:34:53,091 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:53,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T12:34:53,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:53,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834153091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834153093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834153093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834153096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834153096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/bf81751157614ce793d94ffa2f4f08d6 2024-12-10T12:34:53,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/143eb4af6da94f1496d199537c58fae1 is 50, key is test_row_0/B:col10/1733834092476/Put/seqid=0 2024-12-10T12:34:53,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742122_1298 (size=12301) 2024-12-10T12:34:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-10T12:34:53,246 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:53,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T12:34:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834153394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834153394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834153395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834153397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,399 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:53,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T12:34:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834153400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-10T12:34:53,551 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:53,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T12:34:53,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:53,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/143eb4af6da94f1496d199537c58fae1 2024-12-10T12:34:53,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/ad398b47fe7c4bd6acd4bd17c678d66a is 50, key is test_row_0/C:col10/1733834092476/Put/seqid=0 2024-12-10T12:34:53,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742123_1299 (size=12301) 2024-12-10T12:34:53,703 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:53,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T12:34:53,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:53,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,856 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:53,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T12:34:53,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:53,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:53,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:53,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834153897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834153899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834153900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834153901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:53,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834153901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:54,008 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:54,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T12:34:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:54,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:54,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:54,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/ad398b47fe7c4bd6acd4bd17c678d66a 2024-12-10T12:34:54,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/bf81751157614ce793d94ffa2f4f08d6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/bf81751157614ce793d94ffa2f4f08d6 2024-12-10T12:34:54,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/bf81751157614ce793d94ffa2f4f08d6, entries=200, sequenceid=318, filesize=14.4 K 2024-12-10T12:34:54,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/143eb4af6da94f1496d199537c58fae1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/143eb4af6da94f1496d199537c58fae1 2024-12-10T12:34:54,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/143eb4af6da94f1496d199537c58fae1, entries=150, sequenceid=318, filesize=12.0 K 2024-12-10T12:34:54,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/ad398b47fe7c4bd6acd4bd17c678d66a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ad398b47fe7c4bd6acd4bd17c678d66a 2024-12-10T12:34:54,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ad398b47fe7c4bd6acd4bd17c678d66a, entries=150, sequenceid=318, filesize=12.0 K 2024-12-10T12:34:54,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=33.54 KB/34350 for 430faea91f05d6420a9ced51a112393c in 1245ms, sequenceid=318, compaction requested=true 2024-12-10T12:34:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:54,027 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:54,027 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:54,028 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:54,028 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:54,028 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/A is initiating minor compaction (all files) 2024-12-10T12:34:54,028 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/B is initiating minor compaction (all files) 2024-12-10T12:34:54,028 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/A in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:54,028 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/B in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:54,028 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/df67994a5d97487f833c844dc1a2cfdd, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/903cc2f0bea14bc0872f52022457b4c0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/143eb4af6da94f1496d199537c58fae1] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=36.6 K 2024-12-10T12:34:54,028 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ef340ea21fe649c6a8b8270d6cb91ae4, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/4ce7f090a1ae4a51a52388196e01f956, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/bf81751157614ce793d94ffa2f4f08d6] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=39.0 K 2024-12-10T12:34:54,029 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting ef340ea21fe649c6a8b8270d6cb91ae4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733834091237 2024-12-10T12:34:54,029 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4ce7f090a1ae4a51a52388196e01f956, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733834091849 2024-12-10T12:34:54,029 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting df67994a5d97487f833c844dc1a2cfdd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733834091237 2024-12-10T12:34:54,029 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 903cc2f0bea14bc0872f52022457b4c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733834091849 2024-12-10T12:34:54,029 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting bf81751157614ce793d94ffa2f4f08d6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733834092476 2024-12-10T12:34:54,030 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 143eb4af6da94f1496d199537c58fae1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733834092476 2024-12-10T12:34:54,037 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:54,037 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:54,038 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/586feace6e994a0ea1128d1e12f878c0 is 50, key is test_row_0/B:col10/1733834092476/Put/seqid=0 2024-12-10T12:34:54,038 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/f499cfd0e66f44d0aee7b466ab175f2b is 50, key is test_row_0/A:col10/1733834092476/Put/seqid=0 2024-12-10T12:34:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-10T12:34:54,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742125_1301 (size=13017) 2024-12-10T12:34:54,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742124_1300 (size=13017) 2024-12-10T12:34:54,160 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:54,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T12:34:54,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:54,161 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-10T12:34:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:54,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/36dac80e99c84d869e0c88376570f216 is 50, key is test_row_0/A:col10/1733834092785/Put/seqid=0 2024-12-10T12:34:54,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742126_1302 (size=12301) 2024-12-10T12:34:54,456 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/f499cfd0e66f44d0aee7b466ab175f2b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/f499cfd0e66f44d0aee7b466ab175f2b 2024-12-10T12:34:54,456 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/586feace6e994a0ea1128d1e12f878c0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/586feace6e994a0ea1128d1e12f878c0 2024-12-10T12:34:54,463 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/B of 430faea91f05d6420a9ced51a112393c into 586feace6e994a0ea1128d1e12f878c0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:54,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:54,463 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/B, priority=13, startTime=1733834094027; duration=0sec 2024-12-10T12:34:54,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:54,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:B 2024-12-10T12:34:54,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:54,463 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/A of 430faea91f05d6420a9ced51a112393c into f499cfd0e66f44d0aee7b466ab175f2b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:54,463 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:54,463 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/A, priority=13, startTime=1733834094027; duration=0sec 2024-12-10T12:34:54,464 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:54,464 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:A 2024-12-10T12:34:54,465 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:54,465 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/C is initiating minor compaction (all files) 2024-12-10T12:34:54,465 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/C in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:54,465 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/73c0040c8c7f4c8c8af424b9d52c0767, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/117f70b5a38e43a79b769c9424d1be35, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ad398b47fe7c4bd6acd4bd17c678d66a] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=36.6 K 2024-12-10T12:34:54,466 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 73c0040c8c7f4c8c8af424b9d52c0767, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733834091237 2024-12-10T12:34:54,466 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 117f70b5a38e43a79b769c9424d1be35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733834091849 2024-12-10T12:34:54,466 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ad398b47fe7c4bd6acd4bd17c678d66a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733834092476 2024-12-10T12:34:54,489 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:54,490 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/ac141587d76349d0a9761184fc55046e is 50, key is test_row_0/C:col10/1733834092476/Put/seqid=0 2024-12-10T12:34:54,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742127_1303 (size=13017) 2024-12-10T12:34:54,582 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/36dac80e99c84d869e0c88376570f216 2024-12-10T12:34:54,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/d5fbbdcaee984a299e78b681485f92dc is 50, key is test_row_0/B:col10/1733834092785/Put/seqid=0 2024-12-10T12:34:54,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742128_1304 (size=12301) 2024-12-10T12:34:54,903 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/ac141587d76349d0a9761184fc55046e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ac141587d76349d0a9761184fc55046e 2024-12-10T12:34:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:54,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:54,908 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/C of 430faea91f05d6420a9ced51a112393c into ac141587d76349d0a9761184fc55046e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:54,908 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:54,908 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/C, priority=13, startTime=1733834094027; duration=0sec 2024-12-10T12:34:54,908 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:54,908 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:C 2024-12-10T12:34:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834154917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834154917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834154917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:54,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834154920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:54,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834154920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:54,995 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/d5fbbdcaee984a299e78b681485f92dc 2024-12-10T12:34:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/7841fcdd45974889bc5172e90f25ccc7 is 50, key is test_row_0/C:col10/1733834092785/Put/seqid=0 2024-12-10T12:34:55,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742129_1305 (size=12301) 2024-12-10T12:34:55,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834155021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834155021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834155021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834155023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834155023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-10T12:34:55,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834155223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834155223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834155224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834155227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834155227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,404 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/7841fcdd45974889bc5172e90f25ccc7 2024-12-10T12:34:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/36dac80e99c84d869e0c88376570f216 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/36dac80e99c84d869e0c88376570f216 2024-12-10T12:34:55,411 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/36dac80e99c84d869e0c88376570f216, entries=150, sequenceid=326, filesize=12.0 K 2024-12-10T12:34:55,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/d5fbbdcaee984a299e78b681485f92dc as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d5fbbdcaee984a299e78b681485f92dc 2024-12-10T12:34:55,415 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d5fbbdcaee984a299e78b681485f92dc, entries=150, sequenceid=326, filesize=12.0 K 2024-12-10T12:34:55,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/7841fcdd45974889bc5172e90f25ccc7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/7841fcdd45974889bc5172e90f25ccc7 2024-12-10T12:34:55,420 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/7841fcdd45974889bc5172e90f25ccc7, entries=150, sequenceid=326, filesize=12.0 K 2024-12-10T12:34:55,420 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3040): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 430faea91f05d6420a9ced51a112393c in 1260ms, sequenceid=326, compaction requested=false 2024-12-10T12:34:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-10T12:34:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=90 2024-12-10T12:34:55,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=90, resume processing ppid=89 2024-12-10T12:34:55,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, ppid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4820 sec 2024-12-10T12:34:55,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.4850 sec 2024-12-10T12:34:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:55,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-12-10T12:34:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:55,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:55,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834155528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834155528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834155529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/7073810e23a447d7b801a2221219de76 is 50, key is test_row_0/A:col10/1733834095528/Put/seqid=0 2024-12-10T12:34:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834155530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834155531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742130_1306 (size=17181) 2024-12-10T12:34:55,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834155631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834155633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834155633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834155835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834155835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834155835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:55,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/7073810e23a447d7b801a2221219de76 2024-12-10T12:34:55,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/b4eb8f71d4124101add1e7e3d5f209e9 is 50, key is test_row_0/B:col10/1733834095528/Put/seqid=0 2024-12-10T12:34:55,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742131_1307 (size=12301) 2024-12-10T12:34:56,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834156033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:56,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834156034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:56,112 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T12:34:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834156137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834156137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:56,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834156137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:56,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/b4eb8f71d4124101add1e7e3d5f209e9 2024-12-10T12:34:56,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5e95de79742746d8b42246aa68cacfe4 is 50, key is test_row_0/C:col10/1733834095528/Put/seqid=0 2024-12-10T12:34:56,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742132_1308 (size=12301) 2024-12-10T12:34:56,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834156639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:56,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834156639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:56,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834156642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:56,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5e95de79742746d8b42246aa68cacfe4 2024-12-10T12:34:56,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/7073810e23a447d7b801a2221219de76 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7073810e23a447d7b801a2221219de76 2024-12-10T12:34:56,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7073810e23a447d7b801a2221219de76, entries=250, sequenceid=361, filesize=16.8 K 2024-12-10T12:34:56,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/b4eb8f71d4124101add1e7e3d5f209e9 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b4eb8f71d4124101add1e7e3d5f209e9 2024-12-10T12:34:56,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b4eb8f71d4124101add1e7e3d5f209e9, entries=150, sequenceid=361, filesize=12.0 K 2024-12-10T12:34:56,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/5e95de79742746d8b42246aa68cacfe4 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5e95de79742746d8b42246aa68cacfe4 2024-12-10T12:34:56,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5e95de79742746d8b42246aa68cacfe4, entries=150, sequenceid=361, filesize=12.0 K 2024-12-10T12:34:56,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for 430faea91f05d6420a9ced51a112393c in 1275ms, sequenceid=361, compaction requested=true 2024-12-10T12:34:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:34:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:56,803 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:34:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:56,803 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430faea91f05d6420a9ced51a112393c:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:34:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:56,804 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:56,804 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/A is initiating minor compaction (all files) 2024-12-10T12:34:56,804 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/A in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:56,805 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/f499cfd0e66f44d0aee7b466ab175f2b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/36dac80e99c84d869e0c88376570f216, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7073810e23a447d7b801a2221219de76] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=41.5 K 2024-12-10T12:34:56,805 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:56,805 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/B is initiating minor compaction (all files) 2024-12-10T12:34:56,805 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/B in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:56,805 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/586feace6e994a0ea1128d1e12f878c0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d5fbbdcaee984a299e78b681485f92dc, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b4eb8f71d4124101add1e7e3d5f209e9] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=36.7 K 2024-12-10T12:34:56,805 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f499cfd0e66f44d0aee7b466ab175f2b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733834092476 2024-12-10T12:34:56,805 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 586feace6e994a0ea1128d1e12f878c0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733834092476 2024-12-10T12:34:56,805 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 36dac80e99c84d869e0c88376570f216, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733834092784 2024-12-10T12:34:56,805 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting d5fbbdcaee984a299e78b681485f92dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733834092784 2024-12-10T12:34:56,806 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7073810e23a447d7b801a2221219de76, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733834094916 2024-12-10T12:34:56,806 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting b4eb8f71d4124101add1e7e3d5f209e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733834095526 2024-12-10T12:34:56,813 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:56,813 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/9aac32c194dd40569846376e5dd6d6e7 is 50, key is test_row_0/B:col10/1733834095528/Put/seqid=0 2024-12-10T12:34:56,816 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#A#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:56,816 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/a34c0fef784e467d8009f466fa791b4a is 50, key is test_row_0/A:col10/1733834095528/Put/seqid=0 2024-12-10T12:34:56,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742133_1309 (size=13119) 2024-12-10T12:34:56,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742134_1310 (size=13119) 2024-12-10T12:34:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-10T12:34:57,043 INFO [Thread-1102 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 89 completed 2024-12-10T12:34:57,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:34:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-10T12:34:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-10T12:34:57,045 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:34:57,045 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:34:57,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:34:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:57,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:34:57,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:57,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:57,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:57,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:57,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:57,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:57,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/7e60013f21f2474aa36497dd45b567ba is 50, key is test_row_0/A:col10/1733834097048/Put/seqid=0 2024-12-10T12:34:57,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742135_1311 (size=14741) 2024-12-10T12:34:57,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834157082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834157083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-10T12:34:57,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834157184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834157185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,197 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:57,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T12:34:57,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:57,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,199 DEBUG [Thread-1105 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x605827c9 to 127.0.0.1:64063 2024-12-10T12:34:57,199 DEBUG [Thread-1107 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3677bd4f to 127.0.0.1:64063 2024-12-10T12:34:57,199 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:57,199 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:57,200 DEBUG [Thread-1109 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x521aad6f to 127.0.0.1:64063 2024-12-10T12:34:57,200 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:57,202 DEBUG [Thread-1103 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c299cfb to 127.0.0.1:64063 2024-12-10T12:34:57,202 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:57,202 DEBUG [Thread-1111 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f5b2180 to 127.0.0.1:64063 2024-12-10T12:34:57,202 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:57,226 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/9aac32c194dd40569846376e5dd6d6e7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/9aac32c194dd40569846376e5dd6d6e7 2024-12-10T12:34:57,232 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/B of 430faea91f05d6420a9ced51a112393c into 9aac32c194dd40569846376e5dd6d6e7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:57,233 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:57,233 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/B, priority=13, startTime=1733834096803; duration=0sec 2024-12-10T12:34:57,233 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:34:57,233 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:B 2024-12-10T12:34:57,233 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:34:57,234 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:34:57,234 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 430faea91f05d6420a9ced51a112393c/C is initiating minor compaction (all files) 2024-12-10T12:34:57,234 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 430faea91f05d6420a9ced51a112393c/C in TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,234 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ac141587d76349d0a9761184fc55046e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/7841fcdd45974889bc5172e90f25ccc7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5e95de79742746d8b42246aa68cacfe4] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp, totalSize=36.7 K 2024-12-10T12:34:57,235 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ac141587d76349d0a9761184fc55046e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733834092476 2024-12-10T12:34:57,235 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 7841fcdd45974889bc5172e90f25ccc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733834092784 2024-12-10T12:34:57,235 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5e95de79742746d8b42246aa68cacfe4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733834095526 2024-12-10T12:34:57,243 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/a34c0fef784e467d8009f466fa791b4a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/a34c0fef784e467d8009f466fa791b4a 2024-12-10T12:34:57,243 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430faea91f05d6420a9ced51a112393c#C#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:34:57,244 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/75c695e3a6984c7387223d3f18b1d767 is 50, key is test_row_0/C:col10/1733834095528/Put/seqid=0 2024-12-10T12:34:57,248 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/A of 430faea91f05d6420a9ced51a112393c into a34c0fef784e467d8009f466fa791b4a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:57,248 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:57,248 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/A, priority=13, startTime=1733834096803; duration=0sec 2024-12-10T12:34:57,249 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:57,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742136_1312 (size=13119) 2024-12-10T12:34:57,249 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:A 2024-12-10T12:34:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-10T12:34:57,349 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:57,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T12:34:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834157389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834157390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/7e60013f21f2474aa36497dd45b567ba 2024-12-10T12:34:57,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/1d5d6d74996e4d18b86d81a2ff2ea3cd is 50, key is test_row_0/B:col10/1733834097048/Put/seqid=0 2024-12-10T12:34:57,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742137_1313 (size=12301) 2024-12-10T12:34:57,503 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:57,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T12:34:57,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:57,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1733834157643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52836 deadline: 1733834157644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52874 deadline: 1733834157645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-10T12:34:57,655 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/75c695e3a6984c7387223d3f18b1d767 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/75c695e3a6984c7387223d3f18b1d767 2024-12-10T12:34:57,656 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:57,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T12:34:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,663 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 430faea91f05d6420a9ced51a112393c/C of 430faea91f05d6420a9ced51a112393c into 75c695e3a6984c7387223d3f18b1d767(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:34:57,663 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:57,663 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c., storeName=430faea91f05d6420a9ced51a112393c/C, priority=13, startTime=1733834096803; duration=0sec 2024-12-10T12:34:57,663 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:34:57,663 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430faea91f05d6420a9ced51a112393c:C 2024-12-10T12:34:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834157693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834157694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:57,810 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:57,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T12:34:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/1d5d6d74996e4d18b86d81a2ff2ea3cd 2024-12-10T12:34:57,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/fcb23bff426446b3b30f25120e6e55a5 is 50, key is test_row_0/C:col10/1733834097048/Put/seqid=0 2024-12-10T12:34:57,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742138_1314 (size=12301) 2024-12-10T12:34:57,967 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:57,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T12:34:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:57,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:58,120 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:58,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T12:34:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:58,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:58,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-10T12:34:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52838 deadline: 1733834158196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:58,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:34:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52904 deadline: 1733834158199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 2024-12-10T12:34:58,276 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:58,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T12:34:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:58,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:58,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:58,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:58,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:34:58,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/fcb23bff426446b3b30f25120e6e55a5 2024-12-10T12:34:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/7e60013f21f2474aa36497dd45b567ba as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7e60013f21f2474aa36497dd45b567ba 2024-12-10T12:34:58,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7e60013f21f2474aa36497dd45b567ba, entries=200, sequenceid=373, filesize=14.4 K 2024-12-10T12:34:58,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/1d5d6d74996e4d18b86d81a2ff2ea3cd as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1d5d6d74996e4d18b86d81a2ff2ea3cd 2024-12-10T12:34:58,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1d5d6d74996e4d18b86d81a2ff2ea3cd, entries=150, sequenceid=373, filesize=12.0 K 2024-12-10T12:34:58,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/fcb23bff426446b3b30f25120e6e55a5 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/fcb23bff426446b3b30f25120e6e55a5 2024-12-10T12:34:58,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/fcb23bff426446b3b30f25120e6e55a5, entries=150, sequenceid=373, filesize=12.0 K 2024-12-10T12:34:58,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 430faea91f05d6420a9ced51a112393c in 1285ms, sequenceid=373, compaction requested=false 2024-12-10T12:34:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:58,432 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:34:58,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T12:34:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:58,433 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-10T12:34:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:34:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:34:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:34:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:34:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/f50b784c1b64422d8fd6cc9833b3c375 is 50, key is test_row_0/A:col10/1733834097082/Put/seqid=0 2024-12-10T12:34:58,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742139_1315 (size=12301) 2024-12-10T12:34:58,846 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/f50b784c1b64422d8fd6cc9833b3c375 2024-12-10T12:34:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/dda36338c86b4e6c895bd4c67733ff1a is 50, key is test_row_0/B:col10/1733834097082/Put/seqid=0 2024-12-10T12:34:58,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742140_1316 (size=12301) 2024-12-10T12:34:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-10T12:34:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 430faea91f05d6420a9ced51a112393c 2024-12-10T12:34:59,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. as already flushing 2024-12-10T12:34:59,206 DEBUG [Thread-1096 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2fef31f8 to 127.0.0.1:64063 2024-12-10T12:34:59,206 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:59,212 DEBUG [Thread-1098 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0eb04aeb to 127.0.0.1:64063 2024-12-10T12:34:59,212 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:59,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/dda36338c86b4e6c895bd4c67733ff1a 2024-12-10T12:34:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/eba624379c0940a8988889bb49ce5dcd is 50, key is test_row_0/C:col10/1733834097082/Put/seqid=0 2024-12-10T12:34:59,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742141_1317 (size=12301) 2024-12-10T12:34:59,650 DEBUG [Thread-1092 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x34cb3991 to 127.0.0.1:64063 2024-12-10T12:34:59,650 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:59,656 DEBUG [Thread-1100 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6a0e9c8f to 127.0.0.1:64063 2024-12-10T12:34:59,656 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:59,660 DEBUG [Thread-1094 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e9ae050 to 127.0.0.1:64063 2024-12-10T12:34:59,660 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:34:59,683 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/eba624379c0940a8988889bb49ce5dcd 2024-12-10T12:34:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/f50b784c1b64422d8fd6cc9833b3c375 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/f50b784c1b64422d8fd6cc9833b3c375 2024-12-10T12:34:59,694 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/f50b784c1b64422d8fd6cc9833b3c375, entries=150, sequenceid=400, filesize=12.0 K 2024-12-10T12:34:59,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/dda36338c86b4e6c895bd4c67733ff1a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/dda36338c86b4e6c895bd4c67733ff1a 2024-12-10T12:34:59,698 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/dda36338c86b4e6c895bd4c67733ff1a, entries=150, sequenceid=400, filesize=12.0 K 2024-12-10T12:34:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/eba624379c0940a8988889bb49ce5dcd as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/eba624379c0940a8988889bb49ce5dcd 2024-12-10T12:34:59,702 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/eba624379c0940a8988889bb49ce5dcd, entries=150, sequenceid=400, filesize=12.0 K 2024-12-10T12:34:59,703 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=33.54 KB/34350 for 430faea91f05d6420a9ced51a112393c in 1270ms, sequenceid=400, compaction requested=true 2024-12-10T12:34:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2538): Flush status journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:34:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:34:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-10T12:34:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=92 2024-12-10T12:34:59,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-12-10T12:34:59,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6590 sec 2024-12-10T12:34:59,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.6620 sec 2024-12-10T12:35:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-10T12:35:01,154 INFO [Thread-1102 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 91 completed 2024-12-10T12:35:01,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T12:35:01,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 9249 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 8951 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 8767 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 9274 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 8963 2024-12-10T12:35:01,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T12:35:01,155 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-10T12:35:01,155 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7c480dfb to 127.0.0.1:64063 2024-12-10T12:35:01,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:01,157 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-12-10T12:35:01,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T12:35:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=93, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-10T12:35:01,163 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834101163"}]},"ts":"1733834101163"} 2024-12-10T12:35:01,164 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T12:35:01,215 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T12:35:01,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T12:35:01,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=430faea91f05d6420a9ced51a112393c, UNASSIGN}] 2024-12-10T12:35:01,220 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=430faea91f05d6420a9ced51a112393c, UNASSIGN 2024-12-10T12:35:01,221 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=95 updating hbase:meta row=430faea91f05d6420a9ced51a112393c, regionState=CLOSING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:01,223 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T12:35:01,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; CloseRegionProcedure 430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-10T12:35:01,376 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:01,377 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] handler.UnassignRegionHandler(124): Close 430faea91f05d6420a9ced51a112393c 2024-12-10T12:35:01,377 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-12-10T12:35:01,377 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1681): Closing 430faea91f05d6420a9ced51a112393c, disabling compactions & flushes 2024-12-10T12:35:01,377 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:35:01,378 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:35:01,378 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. after waiting 0 ms 2024-12-10T12:35:01,378 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:35:01,378 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(2837): Flushing 430faea91f05d6420a9ced51a112393c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-10T12:35:01,378 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=A 2024-12-10T12:35:01,379 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:01,379 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=B 2024-12-10T12:35:01,379 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:01,379 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 430faea91f05d6420a9ced51a112393c, store=C 2024-12-10T12:35:01,379 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:01,389 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/575d36f58f09499f9eff229a7a3c057c is 50, key is test_row_0/A:col10/1733834099659/Put/seqid=0 2024-12-10T12:35:01,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742142_1318 (size=9857) 2024-12-10T12:35:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-10T12:35:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-10T12:35:01,795 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/575d36f58f09499f9eff229a7a3c057c 2024-12-10T12:35:01,808 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/3416dcdab5bc4dfda1af69a3a3d0aa89 is 50, key is test_row_0/B:col10/1733834099659/Put/seqid=0 2024-12-10T12:35:01,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742143_1319 (size=9857) 2024-12-10T12:35:02,213 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/3416dcdab5bc4dfda1af69a3a3d0aa89 2024-12-10T12:35:02,228 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/e814f80e04504292ba59d7ecdcbc2c34 is 50, key is test_row_0/C:col10/1733834099659/Put/seqid=0 2024-12-10T12:35:02,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742144_1320 (size=9857) 2024-12-10T12:35:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-10T12:35:02,634 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/e814f80e04504292ba59d7ecdcbc2c34 2024-12-10T12:35:02,643 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/A/575d36f58f09499f9eff229a7a3c057c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/575d36f58f09499f9eff229a7a3c057c 2024-12-10T12:35:02,650 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/575d36f58f09499f9eff229a7a3c057c, entries=100, sequenceid=408, filesize=9.6 K 2024-12-10T12:35:02,651 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/B/3416dcdab5bc4dfda1af69a3a3d0aa89 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3416dcdab5bc4dfda1af69a3a3d0aa89 2024-12-10T12:35:02,657 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3416dcdab5bc4dfda1af69a3a3d0aa89, entries=100, sequenceid=408, filesize=9.6 K 2024-12-10T12:35:02,658 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/.tmp/C/e814f80e04504292ba59d7ecdcbc2c34 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e814f80e04504292ba59d7ecdcbc2c34 2024-12-10T12:35:02,663 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e814f80e04504292ba59d7ecdcbc2c34, entries=100, sequenceid=408, filesize=9.6 K 2024-12-10T12:35:02,664 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(3040): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 430faea91f05d6420a9ced51a112393c in 1286ms, sequenceid=408, compaction requested=true 2024-12-10T12:35:02,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/509b45d5a2cc4e4ba1f35e6a34809508, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/5037aafb2da24f788ee19a872baec3ac, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/dec46d2d26cb4d9b8c2ea8298b629815, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/b69ef1e6ee304ce7b301af7fca9e9099, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/c24615754c2c409b812030030ca8ff17, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/97727dc5b73b41a4a8d59cb6777a0280, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/6934578094ac41fc9e931e82ab1bccaa, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/81eb0810cf454ed2823d3b556718cfb5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/9b5800d74061443bad133e2b6bc91d30, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ba43e64848a044ca843a44edffe44b5a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/d00817b55b064346ab2b4365b3489541, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/159b66813cc34d3b918600f47a542659, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/84eb3a15e78f4aacb0221e23afd53592, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/260261a75d784c0aa99352d78eb6aa5d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/500b446e9ed6452fadcc9751b846e3ec, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/2e9dfff50d6d498ca24aabb1a5668eed, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/e3aaf5507f3345bfb9605007901d14b0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ef340ea21fe649c6a8b8270d6cb91ae4, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/16f5fea2be1748d58e199afa107ccadb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/4ce7f090a1ae4a51a52388196e01f956, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/bf81751157614ce793d94ffa2f4f08d6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/f499cfd0e66f44d0aee7b466ab175f2b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/36dac80e99c84d869e0c88376570f216, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7073810e23a447d7b801a2221219de76] to archive 2024-12-10T12:35:02,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:35:02,668 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/509b45d5a2cc4e4ba1f35e6a34809508 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/509b45d5a2cc4e4ba1f35e6a34809508 2024-12-10T12:35:02,672 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/c24615754c2c409b812030030ca8ff17 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/c24615754c2c409b812030030ca8ff17 2024-12-10T12:35:02,672 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/b69ef1e6ee304ce7b301af7fca9e9099 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/b69ef1e6ee304ce7b301af7fca9e9099 2024-12-10T12:35:02,672 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/97727dc5b73b41a4a8d59cb6777a0280 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/97727dc5b73b41a4a8d59cb6777a0280 2024-12-10T12:35:02,672 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/81eb0810cf454ed2823d3b556718cfb5 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/81eb0810cf454ed2823d3b556718cfb5 2024-12-10T12:35:02,672 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/5037aafb2da24f788ee19a872baec3ac to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/5037aafb2da24f788ee19a872baec3ac 2024-12-10T12:35:02,673 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/6934578094ac41fc9e931e82ab1bccaa to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/6934578094ac41fc9e931e82ab1bccaa 2024-12-10T12:35:02,673 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/dec46d2d26cb4d9b8c2ea8298b629815 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/dec46d2d26cb4d9b8c2ea8298b629815 2024-12-10T12:35:02,674 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/9b5800d74061443bad133e2b6bc91d30 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/9b5800d74061443bad133e2b6bc91d30 2024-12-10T12:35:02,675 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ba43e64848a044ca843a44edffe44b5a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ba43e64848a044ca843a44edffe44b5a 2024-12-10T12:35:02,675 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/d00817b55b064346ab2b4365b3489541 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/d00817b55b064346ab2b4365b3489541 2024-12-10T12:35:02,675 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/84eb3a15e78f4aacb0221e23afd53592 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/84eb3a15e78f4aacb0221e23afd53592 2024-12-10T12:35:02,675 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/159b66813cc34d3b918600f47a542659 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/159b66813cc34d3b918600f47a542659 2024-12-10T12:35:02,675 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/260261a75d784c0aa99352d78eb6aa5d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/260261a75d784c0aa99352d78eb6aa5d 2024-12-10T12:35:02,676 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/500b446e9ed6452fadcc9751b846e3ec to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/500b446e9ed6452fadcc9751b846e3ec 2024-12-10T12:35:02,676 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/2e9dfff50d6d498ca24aabb1a5668eed to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/2e9dfff50d6d498ca24aabb1a5668eed 2024-12-10T12:35:02,677 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/e3aaf5507f3345bfb9605007901d14b0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/e3aaf5507f3345bfb9605007901d14b0 2024-12-10T12:35:02,677 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/16f5fea2be1748d58e199afa107ccadb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/16f5fea2be1748d58e199afa107ccadb 2024-12-10T12:35:02,677 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ef340ea21fe649c6a8b8270d6cb91ae4 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/ef340ea21fe649c6a8b8270d6cb91ae4 2024-12-10T12:35:02,677 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/4ce7f090a1ae4a51a52388196e01f956 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/4ce7f090a1ae4a51a52388196e01f956 2024-12-10T12:35:02,678 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/bf81751157614ce793d94ffa2f4f08d6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/bf81751157614ce793d94ffa2f4f08d6 2024-12-10T12:35:02,678 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/36dac80e99c84d869e0c88376570f216 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/36dac80e99c84d869e0c88376570f216 2024-12-10T12:35:02,678 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/f499cfd0e66f44d0aee7b466ab175f2b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/f499cfd0e66f44d0aee7b466ab175f2b 2024-12-10T12:35:02,678 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7073810e23a447d7b801a2221219de76 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7073810e23a447d7b801a2221219de76 2024-12-10T12:35:02,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b61ccbfd56154c4e8da9333eeb4b5bc3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/00ffddf4b80a40e889d0c99614780ad9, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/47b079ae6f2f4f6da4859823bd8b9704, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d680d8d0cac24da7a5d5dbc6431dc235, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/87e88a8f50f3407690961b0b2e20a5ca, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/15b1ef631b024802a03faf879605e498, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/4152948f39c748f3967099df94546351, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/12e3ac5d27b74208afe2685344fc4e25, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/51e7ad22036d4ce29d160a8cd414ebea, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3fbff05924104de4bf47a56d8e755054, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/f9f4ed66bf1942bfa4c376401f8786a0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1f6549529161485fbd6e76046cf9b6a8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/af5da414cade4290956279ebe7df8703, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ef5f2a52d3a04074a787bae12c4d530b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3e445025cd0c434b9b0243ef77961ef6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/94d5d8d87f2f4bc5b76480887aec80d1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/218f161a0191437da3598386ee9057a1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/df67994a5d97487f833c844dc1a2cfdd, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ff56ccfed9124eb49df27ba921ffa242, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/903cc2f0bea14bc0872f52022457b4c0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/586feace6e994a0ea1128d1e12f878c0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/143eb4af6da94f1496d199537c58fae1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d5fbbdcaee984a299e78b681485f92dc, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b4eb8f71d4124101add1e7e3d5f209e9] to archive 2024-12-10T12:35:02,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:35:02,682 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b61ccbfd56154c4e8da9333eeb4b5bc3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b61ccbfd56154c4e8da9333eeb4b5bc3 2024-12-10T12:35:02,682 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/00ffddf4b80a40e889d0c99614780ad9 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/00ffddf4b80a40e889d0c99614780ad9 2024-12-10T12:35:02,683 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/47b079ae6f2f4f6da4859823bd8b9704 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/47b079ae6f2f4f6da4859823bd8b9704 2024-12-10T12:35:02,683 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/15b1ef631b024802a03faf879605e498 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/15b1ef631b024802a03faf879605e498 2024-12-10T12:35:02,683 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d680d8d0cac24da7a5d5dbc6431dc235 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d680d8d0cac24da7a5d5dbc6431dc235 2024-12-10T12:35:02,683 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/12e3ac5d27b74208afe2685344fc4e25 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/12e3ac5d27b74208afe2685344fc4e25 2024-12-10T12:35:02,683 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/87e88a8f50f3407690961b0b2e20a5ca to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/87e88a8f50f3407690961b0b2e20a5ca 2024-12-10T12:35:02,684 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/4152948f39c748f3967099df94546351 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/4152948f39c748f3967099df94546351 2024-12-10T12:35:02,684 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/51e7ad22036d4ce29d160a8cd414ebea to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/51e7ad22036d4ce29d160a8cd414ebea 2024-12-10T12:35:02,685 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3fbff05924104de4bf47a56d8e755054 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3fbff05924104de4bf47a56d8e755054 2024-12-10T12:35:02,686 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/f9f4ed66bf1942bfa4c376401f8786a0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/f9f4ed66bf1942bfa4c376401f8786a0 2024-12-10T12:35:02,686 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1f6549529161485fbd6e76046cf9b6a8 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1f6549529161485fbd6e76046cf9b6a8 2024-12-10T12:35:02,686 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3e445025cd0c434b9b0243ef77961ef6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3e445025cd0c434b9b0243ef77961ef6 2024-12-10T12:35:02,686 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ef5f2a52d3a04074a787bae12c4d530b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ef5f2a52d3a04074a787bae12c4d530b 2024-12-10T12:35:02,686 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/af5da414cade4290956279ebe7df8703 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/af5da414cade4290956279ebe7df8703 2024-12-10T12:35:02,686 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/94d5d8d87f2f4bc5b76480887aec80d1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/94d5d8d87f2f4bc5b76480887aec80d1 2024-12-10T12:35:02,686 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/218f161a0191437da3598386ee9057a1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/218f161a0191437da3598386ee9057a1 2024-12-10T12:35:02,687 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/df67994a5d97487f833c844dc1a2cfdd to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/df67994a5d97487f833c844dc1a2cfdd 2024-12-10T12:35:02,688 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/903cc2f0bea14bc0872f52022457b4c0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/903cc2f0bea14bc0872f52022457b4c0 2024-12-10T12:35:02,688 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ff56ccfed9124eb49df27ba921ffa242 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/ff56ccfed9124eb49df27ba921ffa242 2024-12-10T12:35:02,688 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/586feace6e994a0ea1128d1e12f878c0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/586feace6e994a0ea1128d1e12f878c0 2024-12-10T12:35:02,688 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/143eb4af6da94f1496d199537c58fae1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/143eb4af6da94f1496d199537c58fae1 2024-12-10T12:35:02,688 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d5fbbdcaee984a299e78b681485f92dc to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/d5fbbdcaee984a299e78b681485f92dc 2024-12-10T12:35:02,688 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b4eb8f71d4124101add1e7e3d5f209e9 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/b4eb8f71d4124101add1e7e3d5f209e9 2024-12-10T12:35:02,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/dcd497da5de2474f99d0fa38f280001a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/4015debc579644e48151a90a8291db02, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5724ce916833435098fb7d8f94e3cbf6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/111f28f4e6f74b718ddcafa17b44d890, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5bcf24a5df10464895bea069d29e1534, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ed781c1740be4a7c91a41ade848b61f8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6c53644b4e8045848b5a20de3bc8bbcc, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/48adb9e95925475a912aa1a6bce659bf, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9f034d3e80b14003b81380d8780dfd03, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/68a55516fa30475697e55b997160242c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9a9861b2ce134d01a8bbfc60e6d746d0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/f9e8725fd8af4301b713565ffc59cf79, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/95cfe91dd5444033a4a0c96770e5ef0a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/26cc1160959b48e58c1e035cd8e81881, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/c71efb11b4514ac4bcdeafb4a99e6ca8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5861437e8aae4747832b39e05ba3026b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6b2993b91fde4624876859bfa40476cd, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/73c0040c8c7f4c8c8af424b9d52c0767, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e74c990e2d8f46d6a5cfd046b1b8a8e0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/117f70b5a38e43a79b769c9424d1be35, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ac141587d76349d0a9761184fc55046e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ad398b47fe7c4bd6acd4bd17c678d66a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/7841fcdd45974889bc5172e90f25ccc7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5e95de79742746d8b42246aa68cacfe4] to archive 2024-12-10T12:35:02,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:35:02,693 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/4015debc579644e48151a90a8291db02 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/4015debc579644e48151a90a8291db02 2024-12-10T12:35:02,693 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5724ce916833435098fb7d8f94e3cbf6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5724ce916833435098fb7d8f94e3cbf6 2024-12-10T12:35:02,693 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/111f28f4e6f74b718ddcafa17b44d890 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/111f28f4e6f74b718ddcafa17b44d890 2024-12-10T12:35:02,693 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5bcf24a5df10464895bea069d29e1534 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5bcf24a5df10464895bea069d29e1534 2024-12-10T12:35:02,693 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/dcd497da5de2474f99d0fa38f280001a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/dcd497da5de2474f99d0fa38f280001a 2024-12-10T12:35:02,693 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6c53644b4e8045848b5a20de3bc8bbcc to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6c53644b4e8045848b5a20de3bc8bbcc 2024-12-10T12:35:02,693 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ed781c1740be4a7c91a41ade848b61f8 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ed781c1740be4a7c91a41ade848b61f8 2024-12-10T12:35:02,694 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/48adb9e95925475a912aa1a6bce659bf to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/48adb9e95925475a912aa1a6bce659bf 2024-12-10T12:35:02,695 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9f034d3e80b14003b81380d8780dfd03 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9f034d3e80b14003b81380d8780dfd03 2024-12-10T12:35:02,695 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9a9861b2ce134d01a8bbfc60e6d746d0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/9a9861b2ce134d01a8bbfc60e6d746d0 2024-12-10T12:35:02,696 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/68a55516fa30475697e55b997160242c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/68a55516fa30475697e55b997160242c 2024-12-10T12:35:02,696 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/c71efb11b4514ac4bcdeafb4a99e6ca8 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/c71efb11b4514ac4bcdeafb4a99e6ca8 2024-12-10T12:35:02,696 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/f9e8725fd8af4301b713565ffc59cf79 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/f9e8725fd8af4301b713565ffc59cf79 2024-12-10T12:35:02,696 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/26cc1160959b48e58c1e035cd8e81881 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/26cc1160959b48e58c1e035cd8e81881 2024-12-10T12:35:02,696 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/95cfe91dd5444033a4a0c96770e5ef0a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/95cfe91dd5444033a4a0c96770e5ef0a 2024-12-10T12:35:02,696 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5861437e8aae4747832b39e05ba3026b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5861437e8aae4747832b39e05ba3026b 2024-12-10T12:35:02,698 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/73c0040c8c7f4c8c8af424b9d52c0767 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/73c0040c8c7f4c8c8af424b9d52c0767 2024-12-10T12:35:02,698 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6b2993b91fde4624876859bfa40476cd to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/6b2993b91fde4624876859bfa40476cd 2024-12-10T12:35:02,698 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ac141587d76349d0a9761184fc55046e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ac141587d76349d0a9761184fc55046e 2024-12-10T12:35:02,698 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e74c990e2d8f46d6a5cfd046b1b8a8e0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e74c990e2d8f46d6a5cfd046b1b8a8e0 2024-12-10T12:35:02,698 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/117f70b5a38e43a79b769c9424d1be35 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/117f70b5a38e43a79b769c9424d1be35 2024-12-10T12:35:02,698 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5e95de79742746d8b42246aa68cacfe4 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/5e95de79742746d8b42246aa68cacfe4 2024-12-10T12:35:02,698 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ad398b47fe7c4bd6acd4bd17c678d66a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/ad398b47fe7c4bd6acd4bd17c678d66a 2024-12-10T12:35:02,698 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/7841fcdd45974889bc5172e90f25ccc7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/7841fcdd45974889bc5172e90f25ccc7 2024-12-10T12:35:02,701 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/recovered.edits/411.seqid, newMaxSeqId=411, maxSeqId=1 2024-12-10T12:35:02,702 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c. 2024-12-10T12:35:02,702 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1635): Region close journal for 430faea91f05d6420a9ced51a112393c: 2024-12-10T12:35:02,703 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] handler.UnassignRegionHandler(170): Closed 430faea91f05d6420a9ced51a112393c 2024-12-10T12:35:02,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=95 updating hbase:meta row=430faea91f05d6420a9ced51a112393c, regionState=CLOSED 2024-12-10T12:35:02,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=96, resume processing ppid=95 2024-12-10T12:35:02,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=96, ppid=95, state=SUCCESS; CloseRegionProcedure 430faea91f05d6420a9ced51a112393c, server=fad905551524,38809,1733834007617 in 1.4810 sec 2024-12-10T12:35:02,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=95, resume processing ppid=94 2024-12-10T12:35:02,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=95, ppid=94, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=430faea91f05d6420a9ced51a112393c, UNASSIGN in 1.4870 sec 2024-12-10T12:35:02,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=94, resume processing ppid=93 2024-12-10T12:35:02,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=94, ppid=93, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4910 sec 2024-12-10T12:35:02,708 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834102708"}]},"ts":"1733834102708"} 2024-12-10T12:35:02,709 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T12:35:02,755 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T12:35:02,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=93, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.5970 sec 2024-12-10T12:35:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-10T12:35:03,271 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 93 completed 2024-12-10T12:35:03,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T12:35:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=97, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:03,276 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=97, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-10T12:35:03,278 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=97, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:03,280 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c 2024-12-10T12:35:03,284 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/recovered.edits] 2024-12-10T12:35:03,290 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7e60013f21f2474aa36497dd45b567ba to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/7e60013f21f2474aa36497dd45b567ba 2024-12-10T12:35:03,290 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/f50b784c1b64422d8fd6cc9833b3c375 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/f50b784c1b64422d8fd6cc9833b3c375 2024-12-10T12:35:03,290 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/575d36f58f09499f9eff229a7a3c057c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/575d36f58f09499f9eff229a7a3c057c 2024-12-10T12:35:03,290 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/a34c0fef784e467d8009f466fa791b4a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/A/a34c0fef784e467d8009f466fa791b4a 2024-12-10T12:35:03,296 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1d5d6d74996e4d18b86d81a2ff2ea3cd to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/1d5d6d74996e4d18b86d81a2ff2ea3cd 2024-12-10T12:35:03,296 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/9aac32c194dd40569846376e5dd6d6e7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/9aac32c194dd40569846376e5dd6d6e7 2024-12-10T12:35:03,296 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/dda36338c86b4e6c895bd4c67733ff1a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/dda36338c86b4e6c895bd4c67733ff1a 2024-12-10T12:35:03,297 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3416dcdab5bc4dfda1af69a3a3d0aa89 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/B/3416dcdab5bc4dfda1af69a3a3d0aa89 2024-12-10T12:35:03,300 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/75c695e3a6984c7387223d3f18b1d767 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/75c695e3a6984c7387223d3f18b1d767 2024-12-10T12:35:03,300 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e814f80e04504292ba59d7ecdcbc2c34 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/e814f80e04504292ba59d7ecdcbc2c34 2024-12-10T12:35:03,300 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/eba624379c0940a8988889bb49ce5dcd to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/eba624379c0940a8988889bb49ce5dcd 2024-12-10T12:35:03,301 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/fcb23bff426446b3b30f25120e6e55a5 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/C/fcb23bff426446b3b30f25120e6e55a5 2024-12-10T12:35:03,303 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/recovered.edits/411.seqid to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c/recovered.edits/411.seqid 2024-12-10T12:35:03,303 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/430faea91f05d6420a9ced51a112393c 2024-12-10T12:35:03,303 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T12:35:03,305 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=97, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:03,308 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T12:35:03,310 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T12:35:03,311 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=97, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:03,311 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T12:35:03,311 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733834103311"}]},"ts":"9223372036854775807"} 2024-12-10T12:35:03,313 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-12-10T12:35:03,313 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 430faea91f05d6420a9ced51a112393c, NAME => 'TestAcidGuarantees,,1733834074906.430faea91f05d6420a9ced51a112393c.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T12:35:03,313 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T12:35:03,313 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733834103313"}]},"ts":"9223372036854775807"} 2024-12-10T12:35:03,314 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-12-10T12:35:03,355 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=97, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:03,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=97, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-12-10T12:35:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-10T12:35:03,379 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 97 completed 2024-12-10T12:35:03,392 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testGetAtomicity Thread=245 (was 244) - Thread LEAK? -, OpenFileDescriptor=451 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 243) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9448 (was 9492) 2024-12-10T12:35:03,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMobScanAtomicity Thread=245, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=9448 2024-12-10T12:35:03,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T12:35:03,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:35:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=98, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:03,408 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T12:35:03,408 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:03,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 98 2024-12-10T12:35:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=98 2024-12-10T12:35:03,409 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T12:35:03,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742145_1321 (size=963) 2024-12-10T12:35:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=98 2024-12-10T12:35:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=98 2024-12-10T12:35:03,821 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 2024-12-10T12:35:03,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742146_1322 (size=53) 2024-12-10T12:35:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=98 2024-12-10T12:35:04,233 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:35:04,233 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing 15f8962506199e1be1a11e7189a213ff, disabling compactions & flushes 2024-12-10T12:35:04,233 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:04,234 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:04,234 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. after waiting 0 ms 2024-12-10T12:35:04,234 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:04,234 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:04,234 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:04,235 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T12:35:04,236 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733834104235"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733834104235"}]},"ts":"1733834104235"} 2024-12-10T12:35:04,237 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-10T12:35:04,238 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T12:35:04,238 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834104238"}]},"ts":"1733834104238"} 2024-12-10T12:35:04,240 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T12:35:04,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, ASSIGN}] 2024-12-10T12:35:04,284 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, ASSIGN 2024-12-10T12:35:04,285 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, ASSIGN; state=OFFLINE, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=false 2024-12-10T12:35:04,436 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=99 updating hbase:meta row=15f8962506199e1be1a11e7189a213ff, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:04,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; OpenRegionProcedure 15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=98 2024-12-10T12:35:04,591 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:04,597 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:04,597 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7285): Opening region: {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:35:04,598 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:04,598 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:35:04,598 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7327): checking encryption for 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:04,598 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7330): checking classloading for 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:04,601 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:04,603 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:04,603 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15f8962506199e1be1a11e7189a213ff columnFamilyName A 2024-12-10T12:35:04,603 DEBUG [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:04,603 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(327): Store=15f8962506199e1be1a11e7189a213ff/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:04,604 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:04,605 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:04,605 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15f8962506199e1be1a11e7189a213ff columnFamilyName B 2024-12-10T12:35:04,605 DEBUG [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:04,606 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(327): Store=15f8962506199e1be1a11e7189a213ff/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:04,606 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:04,607 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:04,607 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15f8962506199e1be1a11e7189a213ff columnFamilyName C 2024-12-10T12:35:04,607 DEBUG [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:04,608 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(327): Store=15f8962506199e1be1a11e7189a213ff/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:04,608 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:04,608 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:04,608 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:04,610 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:35:04,611 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1085): writing seq id for 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:04,612 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:35:04,612 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1102): Opened 15f8962506199e1be1a11e7189a213ff; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64801944, jitterRate=-0.034375786781311035}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:35:04,613 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1001): Region open journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:04,614 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., pid=100, masterSystemTime=1733834104591 2024-12-10T12:35:04,615 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:04,615 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:04,615 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=99 updating hbase:meta row=15f8962506199e1be1a11e7189a213ff, regionState=OPEN, openSeqNum=2, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:04,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=100, resume processing ppid=99 2024-12-10T12:35:04,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=100, ppid=99, state=SUCCESS; OpenRegionProcedure 15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 in 177 msec 2024-12-10T12:35:04,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=99, resume processing ppid=98 2024-12-10T12:35:04,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=99, ppid=98, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, ASSIGN in 335 msec 2024-12-10T12:35:04,619 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T12:35:04,619 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834104619"}]},"ts":"1733834104619"} 2024-12-10T12:35:04,619 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T12:35:04,630 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T12:35:04,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=98, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2240 sec 2024-12-10T12:35:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=98 2024-12-10T12:35:05,519 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 98 completed 2024-12-10T12:35:05,523 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1dc42ea6 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@62f74604 2024-12-10T12:35:05,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ec15031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:05,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:05,572 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58766, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:05,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T12:35:05,576 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56676, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T12:35:05,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T12:35:05,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$14(2798): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:35:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=101, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:05,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742147_1323 (size=999) 2024-12-10T12:35:05,991 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.963 2024-12-10T12:35:05,991 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.999 2024-12-10T12:35:05,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T12:35:06,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, REOPEN/MOVE}] 2024-12-10T12:35:06,001 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, REOPEN/MOVE 2024-12-10T12:35:06,001 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=103 updating hbase:meta row=15f8962506199e1be1a11e7189a213ff, regionState=CLOSING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:06,002 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T12:35:06,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE; CloseRegionProcedure 15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:06,154 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:06,155 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(124): Close 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,155 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-12-10T12:35:06,155 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1681): Closing 15f8962506199e1be1a11e7189a213ff, disabling compactions & flushes 2024-12-10T12:35:06,155 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,155 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,156 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. after waiting 0 ms 2024-12-10T12:35:06,156 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,164 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T12:35:06,165 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,165 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1635): Region close journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:06,166 WARN [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionServer(3786): Not adding moved region record: 15f8962506199e1be1a11e7189a213ff to self. 2024-12-10T12:35:06,168 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(170): Closed 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,169 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=103 updating hbase:meta row=15f8962506199e1be1a11e7189a213ff, regionState=CLOSED 2024-12-10T12:35:06,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=104, resume processing ppid=103 2024-12-10T12:35:06,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=104, ppid=103, state=SUCCESS; CloseRegionProcedure 15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 in 168 msec 2024-12-10T12:35:06,174 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, REOPEN/MOVE; state=CLOSED, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=true 2024-12-10T12:35:06,324 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=103 updating hbase:meta row=15f8962506199e1be1a11e7189a213ff, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:06,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=105, ppid=103, state=RUNNABLE; OpenRegionProcedure 15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:06,480 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:06,487 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,487 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(7285): Opening region: {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:35:06,489 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,489 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:35:06,489 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(7327): checking encryption for 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,489 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(7330): checking classloading for 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,492 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,493 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:06,493 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15f8962506199e1be1a11e7189a213ff columnFamilyName A 2024-12-10T12:35:06,495 DEBUG [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:06,496 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(327): Store=15f8962506199e1be1a11e7189a213ff/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:06,496 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,497 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:06,497 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15f8962506199e1be1a11e7189a213ff columnFamilyName B 2024-12-10T12:35:06,497 DEBUG [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:06,498 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(327): Store=15f8962506199e1be1a11e7189a213ff/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:06,498 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,499 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:06,499 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15f8962506199e1be1a11e7189a213ff columnFamilyName C 2024-12-10T12:35:06,499 DEBUG [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:06,500 INFO [StoreOpener-15f8962506199e1be1a11e7189a213ff-1 {}] regionserver.HStore(327): Store=15f8962506199e1be1a11e7189a213ff/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:06,500 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,501 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,503 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,505 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:35:06,507 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(1085): writing seq id for 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,508 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(1102): Opened 15f8962506199e1be1a11e7189a213ff; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62083408, jitterRate=-0.07488512992858887}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:35:06,509 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(1001): Region open journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:06,510 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., pid=105, masterSystemTime=1733834106480 2024-12-10T12:35:06,512 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,512 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,513 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=103 updating hbase:meta row=15f8962506199e1be1a11e7189a213ff, regionState=OPEN, openSeqNum=5, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:06,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=105, resume processing ppid=103 2024-12-10T12:35:06,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=105, ppid=103, state=SUCCESS; OpenRegionProcedure 15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 in 188 msec 2024-12-10T12:35:06,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=103, resume processing ppid=102 2024-12-10T12:35:06,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=103, ppid=102, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, REOPEN/MOVE in 516 msec 2024-12-10T12:35:06,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=102, resume processing ppid=101 2024-12-10T12:35:06,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=102, ppid=101, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-12-10T12:35:06,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=101, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 945 msec 2024-12-10T12:35:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-12-10T12:35:06,527 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x117e86d9 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@49e13594 2024-12-10T12:35:06,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd5b441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,589 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6cd96549 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c54a0d3 2024-12-10T12:35:06,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c336ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,599 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31aea41b to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3875c8c5 2024-12-10T12:35:06,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f94d721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,607 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1e247aa1 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@801ba40 2024-12-10T12:35:06,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319559be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,616 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2205f666 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27539bdc 2024-12-10T12:35:06,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c907e21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,624 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5e3203d9 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61ec0f48 2024-12-10T12:35:06,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e4d3d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,633 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x798e7fd4 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7819b9e2 2024-12-10T12:35:06,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b308f62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,642 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7284f16d to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@47679076 2024-12-10T12:35:06,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68035c67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,650 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x37a637ac to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4cb9e50e 2024-12-10T12:35:06,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eab689a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,657 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3fa53591 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3cb726fe 2024-12-10T12:35:06,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59bd764a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:06,668 DEBUG [hconnection-0x562f45bd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,668 DEBUG [hconnection-0x7302f8fd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,668 DEBUG [hconnection-0x4510a4d1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,668 DEBUG [hconnection-0x1dc23094-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,669 DEBUG [hconnection-0x49347c6a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,669 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58770, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,669 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58772, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,669 DEBUG [hconnection-0x6077667c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,669 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58778, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,670 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58790, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,670 DEBUG [hconnection-0x54e61ba6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:06,670 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58798, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,670 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58800, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,671 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58804, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=106, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees 2024-12-10T12:35:06,671 DEBUG [hconnection-0x23880b4c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,672 DEBUG [hconnection-0x71212b4c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-10T12:35:06,672 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=106, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:06,673 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=106, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:06,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:06,674 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58806, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,674 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58814, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,675 DEBUG [hconnection-0x5610f1b3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:06,677 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58820, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:06,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:35:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:06,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834166693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834166693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834166693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834166694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834166694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210033ef0dda15346609f8507345b7b9b36_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834106677/Put/seqid=0 2024-12-10T12:35:06,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742148_1324 (size=12154) 2024-12-10T12:35:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-10T12:35:06,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834166794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834166794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834166794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834166795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834166795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,824 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:06,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-10T12:35:06,976 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:06,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:06,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:06,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:06,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:06,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:06,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834166997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834166998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834166998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:06,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834166998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834166998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:07,128 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210033ef0dda15346609f8507345b7b9b36_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210033ef0dda15346609f8507345b7b9b36_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:07,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/7216ffd238094eb79a47879a3e19bdba, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:07,129 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:07,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/7216ffd238094eb79a47879a3e19bdba is 175, key is test_row_0/A:col10/1733834106677/Put/seqid=0 2024-12-10T12:35:07,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:07,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:07,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742149_1325 (size=30955) 2024-12-10T12:35:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-10T12:35:07,282 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:07,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:07,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:07,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834167299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834167299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834167300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834167300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834167301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,435 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:07,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,536 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=19, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/7216ffd238094eb79a47879a3e19bdba 2024-12-10T12:35:07,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/06b6e2ac86974e9a949bbe567c5f1693 is 50, key is test_row_0/B:col10/1733834106677/Put/seqid=0 2024-12-10T12:35:07,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742150_1326 (size=12001) 2024-12-10T12:35:07,587 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:07,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:07,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:07,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,739 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:07,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-10T12:35:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834167803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834167804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834167805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834167806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834167808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:07,891 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:07,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:07,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:07,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/06b6e2ac86974e9a949bbe567c5f1693 2024-12-10T12:35:07,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/9a41c89aaa284f208fc3c07db5ad1712 is 50, key is test_row_0/C:col10/1733834106677/Put/seqid=0 2024-12-10T12:35:07,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742151_1327 (size=12001) 2024-12-10T12:35:08,044 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:08,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:08,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:08,196 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:08,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:08,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:08,230 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T12:35:08,348 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:08,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:08,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:08,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/9a41c89aaa284f208fc3c07db5ad1712 2024-12-10T12:35:08,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/7216ffd238094eb79a47879a3e19bdba as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7216ffd238094eb79a47879a3e19bdba 2024-12-10T12:35:08,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7216ffd238094eb79a47879a3e19bdba, entries=150, sequenceid=19, filesize=30.2 K 2024-12-10T12:35:08,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/06b6e2ac86974e9a949bbe567c5f1693 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06b6e2ac86974e9a949bbe567c5f1693 2024-12-10T12:35:08,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06b6e2ac86974e9a949bbe567c5f1693, entries=150, sequenceid=19, filesize=11.7 K 2024-12-10T12:35:08,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/9a41c89aaa284f208fc3c07db5ad1712 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a41c89aaa284f208fc3c07db5ad1712 2024-12-10T12:35:08,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a41c89aaa284f208fc3c07db5ad1712, entries=150, sequenceid=19, filesize=11.7 K 2024-12-10T12:35:08,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for 15f8962506199e1be1a11e7189a213ff in 1722ms, sequenceid=19, compaction requested=false 2024-12-10T12:35:08,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:08,501 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:08,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-12-10T12:35:08,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:08,501 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-10T12:35:08,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:08,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d048e219340a42aabdf1bd67989387ae_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834106688/Put/seqid=0 2024-12-10T12:35:08,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742152_1328 (size=12154) 2024-12-10T12:35:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-10T12:35:08,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:08,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834168820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:08,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834168820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:08,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834168820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:08,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834168821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:08,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834168824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:08,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:08,916 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d048e219340a42aabdf1bd67989387ae_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d048e219340a42aabdf1bd67989387ae_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/f90e565ab1a24a7f89a0372ecb51e0de, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/f90e565ab1a24a7f89a0372ecb51e0de is 175, key is test_row_0/A:col10/1733834106688/Put/seqid=0 2024-12-10T12:35:08,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742153_1329 (size=30955) 2024-12-10T12:35:08,921 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/f90e565ab1a24a7f89a0372ecb51e0de 2024-12-10T12:35:08,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/602f0bfe5aa34945a53a9b63e3341633 is 50, key is test_row_0/B:col10/1733834106688/Put/seqid=0 2024-12-10T12:35:08,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834168925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:08,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834168925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:08,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834168925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:08,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742154_1330 (size=12001) 2024-12-10T12:35:08,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834168926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:08,931 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/602f0bfe5aa34945a53a9b63e3341633 2024-12-10T12:35:08,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/4c81c9ebeb7d4fd1acdf00c9cb6f67a7 is 50, key is test_row_0/C:col10/1733834106688/Put/seqid=0 2024-12-10T12:35:08,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742155_1331 (size=12001) 2024-12-10T12:35:09,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834169128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834169128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834169129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834169132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,341 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/4c81c9ebeb7d4fd1acdf00c9cb6f67a7 2024-12-10T12:35:09,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/f90e565ab1a24a7f89a0372ecb51e0de as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f90e565ab1a24a7f89a0372ecb51e0de 2024-12-10T12:35:09,349 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f90e565ab1a24a7f89a0372ecb51e0de, entries=150, sequenceid=42, filesize=30.2 K 2024-12-10T12:35:09,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/602f0bfe5aa34945a53a9b63e3341633 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/602f0bfe5aa34945a53a9b63e3341633 2024-12-10T12:35:09,353 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/602f0bfe5aa34945a53a9b63e3341633, entries=150, sequenceid=42, filesize=11.7 K 2024-12-10T12:35:09,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/4c81c9ebeb7d4fd1acdf00c9cb6f67a7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/4c81c9ebeb7d4fd1acdf00c9cb6f67a7 2024-12-10T12:35:09,357 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/4c81c9ebeb7d4fd1acdf00c9cb6f67a7, entries=150, sequenceid=42, filesize=11.7 K 2024-12-10T12:35:09,357 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 15f8962506199e1be1a11e7189a213ff in 856ms, sequenceid=42, compaction requested=false 2024-12-10T12:35:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-12-10T12:35:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=107 2024-12-10T12:35:09,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=107, resume processing ppid=106 2024-12-10T12:35:09,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=107, ppid=106, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6860 sec 2024-12-10T12:35:09,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=106, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees in 2.6900 sec 2024-12-10T12:35:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:09,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-10T12:35:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:09,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:09,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:09,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107d59a3c024384e60815265d8767e24ba_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834109434/Put/seqid=0 2024-12-10T12:35:09,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742156_1332 (size=14594) 2024-12-10T12:35:09,449 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:09,454 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107d59a3c024384e60815265d8767e24ba_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107d59a3c024384e60815265d8767e24ba_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:09,455 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/4a3439ad9c0c4f6884766ad2e5632dc5, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:09,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/4a3439ad9c0c4f6884766ad2e5632dc5 is 175, key is test_row_0/A:col10/1733834109434/Put/seqid=0 2024-12-10T12:35:09,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742157_1333 (size=39549) 2024-12-10T12:35:09,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834169465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834169467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834169468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834169469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834169570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834169572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834169573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834169573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834169773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834169776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834169777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834169777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:09,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/4a3439ad9c0c4f6884766ad2e5632dc5 2024-12-10T12:35:09,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/1ea07ba3f81c4042a0f68e9cdc3bba62 is 50, key is test_row_0/B:col10/1733834109434/Put/seqid=0 2024-12-10T12:35:09,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742158_1334 (size=12001) 2024-12-10T12:35:10,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834170076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:10,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834170080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:10,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834170082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:10,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834170082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:10,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/1ea07ba3f81c4042a0f68e9cdc3bba62 2024-12-10T12:35:10,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c7c80e486a864a1f8b8d296be794d5f0 is 50, key is test_row_0/C:col10/1733834109434/Put/seqid=0 2024-12-10T12:35:10,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742159_1335 (size=12001) 2024-12-10T12:35:10,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834170582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:10,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834170587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:10,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834170587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:10,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834170588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:10,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c7c80e486a864a1f8b8d296be794d5f0 2024-12-10T12:35:10,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/4a3439ad9c0c4f6884766ad2e5632dc5 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/4a3439ad9c0c4f6884766ad2e5632dc5 2024-12-10T12:35:10,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/4a3439ad9c0c4f6884766ad2e5632dc5, entries=200, sequenceid=57, filesize=38.6 K 2024-12-10T12:35:10,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/1ea07ba3f81c4042a0f68e9cdc3bba62 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/1ea07ba3f81c4042a0f68e9cdc3bba62 2024-12-10T12:35:10,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/1ea07ba3f81c4042a0f68e9cdc3bba62, entries=150, sequenceid=57, filesize=11.7 K 2024-12-10T12:35:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c7c80e486a864a1f8b8d296be794d5f0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c7c80e486a864a1f8b8d296be794d5f0 2024-12-10T12:35:10,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c7c80e486a864a1f8b8d296be794d5f0, entries=150, sequenceid=57, filesize=11.7 K 2024-12-10T12:35:10,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 15f8962506199e1be1a11e7189a213ff in 1259ms, sequenceid=57, compaction requested=true 2024-12-10T12:35:10,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:10,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:10,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:10,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:10,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:10,694 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:10,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:10,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:10,694 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:10,694 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:10,694 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:10,694 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/B is initiating minor compaction (all files) 2024-12-10T12:35:10,694 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/A is initiating minor compaction (all files) 2024-12-10T12:35:10,694 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/B in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:10,694 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/A in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:10,695 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06b6e2ac86974e9a949bbe567c5f1693, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/602f0bfe5aa34945a53a9b63e3341633, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/1ea07ba3f81c4042a0f68e9cdc3bba62] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=35.2 K 2024-12-10T12:35:10,695 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7216ffd238094eb79a47879a3e19bdba, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f90e565ab1a24a7f89a0372ecb51e0de, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/4a3439ad9c0c4f6884766ad2e5632dc5] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=99.1 K 2024-12-10T12:35:10,695 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:10,695 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7216ffd238094eb79a47879a3e19bdba, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f90e565ab1a24a7f89a0372ecb51e0de, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/4a3439ad9c0c4f6884766ad2e5632dc5] 2024-12-10T12:35:10,695 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 06b6e2ac86974e9a949bbe567c5f1693, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1733834106677 2024-12-10T12:35:10,695 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7216ffd238094eb79a47879a3e19bdba, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1733834106677 2024-12-10T12:35:10,695 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 602f0bfe5aa34945a53a9b63e3341633, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733834106688 2024-12-10T12:35:10,695 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f90e565ab1a24a7f89a0372ecb51e0de, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733834106688 2024-12-10T12:35:10,695 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1ea07ba3f81c4042a0f68e9cdc3bba62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733834108820 2024-12-10T12:35:10,695 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4a3439ad9c0c4f6884766ad2e5632dc5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733834108820 2024-12-10T12:35:10,700 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:10,701 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:10,702 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/f318fc93fe1241fe9d0f5e8a346d126d is 50, key is test_row_0/B:col10/1733834109434/Put/seqid=0 2024-12-10T12:35:10,702 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412109f0cab75b2d241af9aee4893bc1879e4_15f8962506199e1be1a11e7189a213ff store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:10,704 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412109f0cab75b2d241af9aee4893bc1879e4_15f8962506199e1be1a11e7189a213ff, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:10,704 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109f0cab75b2d241af9aee4893bc1879e4_15f8962506199e1be1a11e7189a213ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:10,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742160_1336 (size=12104) 2024-12-10T12:35:10,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742161_1337 (size=4469) 2024-12-10T12:35:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-10T12:35:10,776 INFO [Thread-1478 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 106 completed 2024-12-10T12:35:10,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=108, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees 2024-12-10T12:35:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-10T12:35:10,779 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=108, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:10,779 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=108, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:10,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:10,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-10T12:35:10,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:10,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:10,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:10,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:10,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:10,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:10,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f6a50513e5ae4d37b265e6c27a6725df_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834110841/Put/seqid=0 2024-12-10T12:35:10,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742162_1338 (size=14594) 2024-12-10T12:35:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-10T12:35:10,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834170884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:10,930 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:10,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:10,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:10,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:10,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:10,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:10,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:10,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834170988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-10T12:35:11,083 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:11,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,121 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/f318fc93fe1241fe9d0f5e8a346d126d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/f318fc93fe1241fe9d0f5e8a346d126d 2024-12-10T12:35:11,124 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#A#compaction#285 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:11,125 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/380b5e7653d64d0598a3847322d06dec is 175, key is test_row_0/A:col10/1733834109434/Put/seqid=0 2024-12-10T12:35:11,125 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/B of 15f8962506199e1be1a11e7189a213ff into f318fc93fe1241fe9d0f5e8a346d126d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:11,125 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:11,125 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/B, priority=13, startTime=1733834110694; duration=0sec 2024-12-10T12:35:11,125 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:11,125 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:B 2024-12-10T12:35:11,125 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:11,126 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:11,126 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/C is initiating minor compaction (all files) 2024-12-10T12:35:11,126 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/C in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,126 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a41c89aaa284f208fc3c07db5ad1712, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/4c81c9ebeb7d4fd1acdf00c9cb6f67a7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c7c80e486a864a1f8b8d296be794d5f0] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=35.2 K 2024-12-10T12:35:11,126 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 9a41c89aaa284f208fc3c07db5ad1712, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1733834106677 2024-12-10T12:35:11,127 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 4c81c9ebeb7d4fd1acdf00c9cb6f67a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733834106688 2024-12-10T12:35:11,127 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting c7c80e486a864a1f8b8d296be794d5f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733834108820 2024-12-10T12:35:11,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742163_1339 (size=31058) 2024-12-10T12:35:11,133 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#C#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:11,133 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/9a1176b770d748349227cc9f673bea25 is 50, key is test_row_0/C:col10/1733834109434/Put/seqid=0 2024-12-10T12:35:11,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742164_1340 (size=12104) 2024-12-10T12:35:11,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834171192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:11,235 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:11,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,257 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:11,260 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f6a50513e5ae4d37b265e6c27a6725df_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f6a50513e5ae4d37b265e6c27a6725df_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:11,260 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/5230445b40574f2f8916d6d33da34de7, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:11,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/5230445b40574f2f8916d6d33da34de7 is 175, key is test_row_0/A:col10/1733834110841/Put/seqid=0 2024-12-10T12:35:11,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742165_1341 (size=39549) 2024-12-10T12:35:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-10T12:35:11,388 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:11,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:11,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:11,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834171494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:11,532 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/380b5e7653d64d0598a3847322d06dec as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/380b5e7653d64d0598a3847322d06dec 2024-12-10T12:35:11,536 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/A of 15f8962506199e1be1a11e7189a213ff into 380b5e7653d64d0598a3847322d06dec(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:11,536 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:11,537 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/A, priority=13, startTime=1733834110694; duration=0sec 2024-12-10T12:35:11,537 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:11,537 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:A 2024-12-10T12:35:11,540 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:11,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:11,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:11,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,542 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/9a1176b770d748349227cc9f673bea25 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a1176b770d748349227cc9f673bea25 2024-12-10T12:35:11,546 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/C of 15f8962506199e1be1a11e7189a213ff into 9a1176b770d748349227cc9f673bea25(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:11,546 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:11,546 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/C, priority=13, startTime=1733834110694; duration=0sec 2024-12-10T12:35:11,546 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:11,546 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:C 2024-12-10T12:35:11,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834171590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:11,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834171592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:11,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834171595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:11,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834171602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:11,664 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/5230445b40574f2f8916d6d33da34de7 2024-12-10T12:35:11,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/fca701b5ac7842f6a6b62c3a2c2bf78a is 50, key is test_row_0/B:col10/1733834110841/Put/seqid=0 2024-12-10T12:35:11,692 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:11,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:11,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:11,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742166_1342 (size=12001) 2024-12-10T12:35:11,845 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:11,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:11,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:11,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-10T12:35:11,997 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:11,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:11,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834172000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:12,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/fca701b5ac7842f6a6b62c3a2c2bf78a 2024-12-10T12:35:12,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/1b29ff2d29e1473f8abe227ae1b2d2d4 is 50, key is test_row_0/C:col10/1733834110841/Put/seqid=0 2024-12-10T12:35:12,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742167_1343 (size=12001) 2024-12-10T12:35:12,149 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:12,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:12,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:12,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:12,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:12,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,302 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:12,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:12,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:12,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:12,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:12,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,454 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:12,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:12,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:12,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:12,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:12,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:12,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/1b29ff2d29e1473f8abe227ae1b2d2d4 2024-12-10T12:35:12,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/5230445b40574f2f8916d6d33da34de7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5230445b40574f2f8916d6d33da34de7 2024-12-10T12:35:12,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5230445b40574f2f8916d6d33da34de7, entries=200, sequenceid=79, filesize=38.6 K 2024-12-10T12:35:12,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/fca701b5ac7842f6a6b62c3a2c2bf78a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/fca701b5ac7842f6a6b62c3a2c2bf78a 2024-12-10T12:35:12,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/fca701b5ac7842f6a6b62c3a2c2bf78a, entries=150, sequenceid=79, filesize=11.7 K 2024-12-10T12:35:12,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/1b29ff2d29e1473f8abe227ae1b2d2d4 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1b29ff2d29e1473f8abe227ae1b2d2d4 2024-12-10T12:35:12,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1b29ff2d29e1473f8abe227ae1b2d2d4, entries=150, sequenceid=79, filesize=11.7 K 2024-12-10T12:35:12,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 15f8962506199e1be1a11e7189a213ff in 1679ms, sequenceid=79, compaction requested=false 2024-12-10T12:35:12,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:12,606 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:12,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-12-10T12:35:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:12,607 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-10T12:35:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:12,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121050d532c6b0ab440d852cbf48bead819f_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834110874/Put/seqid=0 2024-12-10T12:35:12,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742168_1344 (size=12154) 2024-12-10T12:35:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-10T12:35:13,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:13,020 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121050d532c6b0ab440d852cbf48bead819f_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121050d532c6b0ab440d852cbf48bead819f_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:13,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/5b1ec42b93e2433c9565be6d9fbbff45, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:13,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/5b1ec42b93e2433c9565be6d9fbbff45 is 175, key is test_row_0/A:col10/1733834110874/Put/seqid=0 2024-12-10T12:35:13,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742169_1345 (size=30955) 2024-12-10T12:35:13,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834173084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:13,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834173185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:13,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834173389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:13,438 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/5b1ec42b93e2433c9565be6d9fbbff45 2024-12-10T12:35:13,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/86176f03bf644f51ac090f53c0544a05 is 50, key is test_row_0/B:col10/1733834110874/Put/seqid=0 2024-12-10T12:35:13,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742170_1346 (size=12001) 2024-12-10T12:35:13,448 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/86176f03bf644f51ac090f53c0544a05 2024-12-10T12:35:13,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/306b221b571246f795bf689e732ec50b is 50, key is test_row_0/C:col10/1733834110874/Put/seqid=0 2024-12-10T12:35:13,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742171_1347 (size=12001) 2024-12-10T12:35:13,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834173596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:13,600 DEBUG [Thread-1470 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4133 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:35:13,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834173602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:13,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834173603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:13,606 DEBUG [Thread-1468 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4141 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:35:13,606 DEBUG [Thread-1472 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4138 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:35:13,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834173618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:13,620 DEBUG [Thread-1474 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4154 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:35:13,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834173693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:13,856 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/306b221b571246f795bf689e732ec50b 2024-12-10T12:35:13,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/5b1ec42b93e2433c9565be6d9fbbff45 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5b1ec42b93e2433c9565be6d9fbbff45 2024-12-10T12:35:13,864 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5b1ec42b93e2433c9565be6d9fbbff45, entries=150, sequenceid=96, filesize=30.2 K 2024-12-10T12:35:13,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/86176f03bf644f51ac090f53c0544a05 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/86176f03bf644f51ac090f53c0544a05 2024-12-10T12:35:13,868 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/86176f03bf644f51ac090f53c0544a05, entries=150, sequenceid=96, filesize=11.7 K 2024-12-10T12:35:13,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/306b221b571246f795bf689e732ec50b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/306b221b571246f795bf689e732ec50b 2024-12-10T12:35:13,871 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/306b221b571246f795bf689e732ec50b, entries=150, sequenceid=96, filesize=11.7 K 2024-12-10T12:35:13,872 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 15f8962506199e1be1a11e7189a213ff in 1265ms, sequenceid=96, compaction requested=true 2024-12-10T12:35:13,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:13,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:13,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=109 2024-12-10T12:35:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=109 2024-12-10T12:35:13,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=109, resume processing ppid=108 2024-12-10T12:35:13,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=109, ppid=108, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0940 sec 2024-12-10T12:35:13,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=108, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees in 3.0970 sec 2024-12-10T12:35:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:14,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-10T12:35:14,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:14,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101cb52d42a0c442c7a1675f3608eefd46_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834113070/Put/seqid=0 2024-12-10T12:35:14,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742172_1348 (size=14594) 2024-12-10T12:35:14,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834174244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:14,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834174348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:14,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834174553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:14,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:14,620 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101cb52d42a0c442c7a1675f3608eefd46_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101cb52d42a0c442c7a1675f3608eefd46_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:14,620 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/d80c0cba52564b0bbf365cb271170979, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:14,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/d80c0cba52564b0bbf365cb271170979 is 175, key is test_row_0/A:col10/1733834113070/Put/seqid=0 2024-12-10T12:35:14,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742173_1349 (size=39549) 2024-12-10T12:35:14,624 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/d80c0cba52564b0bbf365cb271170979 2024-12-10T12:35:14,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/b880da6d2f704b2e8e51265fc3f2868f is 50, key is test_row_0/B:col10/1733834113070/Put/seqid=0 2024-12-10T12:35:14,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742174_1350 (size=12001) 2024-12-10T12:35:14,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834174858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-10T12:35:14,883 INFO [Thread-1478 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 108 completed 2024-12-10T12:35:14,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=110, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees 2024-12-10T12:35:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-10T12:35:14,885 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=110, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:14,885 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=110, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:14,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-10T12:35:15,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/b880da6d2f704b2e8e51265fc3f2868f 2024-12-10T12:35:15,036 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:15,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-12-10T12:35:15,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:15,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:15,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:15,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/7f0db2c207d54b4ab27ba034d66e3f94 is 50, key is test_row_0/C:col10/1733834113070/Put/seqid=0 2024-12-10T12:35:15,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742175_1351 (size=12001) 2024-12-10T12:35:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-10T12:35:15,188 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:15,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-12-10T12:35:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:15,341 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:15,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-12-10T12:35:15,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:15,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834175365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:15,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/7f0db2c207d54b4ab27ba034d66e3f94 2024-12-10T12:35:15,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/d80c0cba52564b0bbf365cb271170979 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d80c0cba52564b0bbf365cb271170979 2024-12-10T12:35:15,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d80c0cba52564b0bbf365cb271170979, entries=200, sequenceid=119, filesize=38.6 K 2024-12-10T12:35:15,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/b880da6d2f704b2e8e51265fc3f2868f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/b880da6d2f704b2e8e51265fc3f2868f 2024-12-10T12:35:15,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/b880da6d2f704b2e8e51265fc3f2868f, entries=150, sequenceid=119, filesize=11.7 K 2024-12-10T12:35:15,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/7f0db2c207d54b4ab27ba034d66e3f94 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/7f0db2c207d54b4ab27ba034d66e3f94 2024-12-10T12:35:15,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/7f0db2c207d54b4ab27ba034d66e3f94, entries=150, sequenceid=119, filesize=11.7 K 2024-12-10T12:35:15,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 15f8962506199e1be1a11e7189a213ff in 1260ms, sequenceid=119, compaction requested=true 2024-12-10T12:35:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:15,462 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:35:15,462 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:35:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:15,463 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:35:15,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:35:15,463 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/A is initiating minor compaction (all files) 2024-12-10T12:35:15,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/B is initiating minor compaction (all files) 2024-12-10T12:35:15,463 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/A in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,463 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/B in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,463 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/f318fc93fe1241fe9d0f5e8a346d126d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/fca701b5ac7842f6a6b62c3a2c2bf78a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/86176f03bf644f51ac090f53c0544a05, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/b880da6d2f704b2e8e51265fc3f2868f] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=47.0 K 2024-12-10T12:35:15,463 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/380b5e7653d64d0598a3847322d06dec, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5230445b40574f2f8916d6d33da34de7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5b1ec42b93e2433c9565be6d9fbbff45, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d80c0cba52564b0bbf365cb271170979] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=137.8 K 2024-12-10T12:35:15,463 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,463 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/380b5e7653d64d0598a3847322d06dec, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5230445b40574f2f8916d6d33da34de7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5b1ec42b93e2433c9565be6d9fbbff45, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d80c0cba52564b0bbf365cb271170979] 2024-12-10T12:35:15,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting f318fc93fe1241fe9d0f5e8a346d126d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733834108820 2024-12-10T12:35:15,463 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 380b5e7653d64d0598a3847322d06dec, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733834108820 2024-12-10T12:35:15,464 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting fca701b5ac7842f6a6b62c3a2c2bf78a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733834109462 2024-12-10T12:35:15,464 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5230445b40574f2f8916d6d33da34de7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733834109462 2024-12-10T12:35:15,464 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5b1ec42b93e2433c9565be6d9fbbff45, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733834110874 2024-12-10T12:35:15,464 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 86176f03bf644f51ac090f53c0544a05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733834110874 2024-12-10T12:35:15,464 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting d80c0cba52564b0bbf365cb271170979, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733834113070 2024-12-10T12:35:15,464 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting b880da6d2f704b2e8e51265fc3f2868f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733834113070 2024-12-10T12:35:15,470 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:15,471 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:15,471 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210d794eca84d1f441b8a8654625eeb8d1e_15f8962506199e1be1a11e7189a213ff store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:15,472 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/eff403eff245497091443ae4792455f7 is 50, key is test_row_0/B:col10/1733834113070/Put/seqid=0 2024-12-10T12:35:15,473 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210d794eca84d1f441b8a8654625eeb8d1e_15f8962506199e1be1a11e7189a213ff, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:15,474 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d794eca84d1f441b8a8654625eeb8d1e_15f8962506199e1be1a11e7189a213ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-10T12:35:15,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742177_1353 (size=12241) 2024-12-10T12:35:15,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742176_1352 (size=4469) 2024-12-10T12:35:15,493 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:15,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-12-10T12:35:15,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,494 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:35:15,494 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#A#compaction#298 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:15,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:15,495 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/aac8926214c14e0ea20688fb8891a29b is 175, key is test_row_0/A:col10/1733834113070/Put/seqid=0 2024-12-10T12:35:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:15,498 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/eff403eff245497091443ae4792455f7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/eff403eff245497091443ae4792455f7 2024-12-10T12:35:15,506 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/B of 15f8962506199e1be1a11e7189a213ff into eff403eff245497091443ae4792455f7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:15,507 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:15,507 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/B, priority=12, startTime=1733834115462; duration=0sec 2024-12-10T12:35:15,507 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:15,507 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:B 2024-12-10T12:35:15,507 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:35:15,509 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:35:15,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742178_1354 (size=31195) 2024-12-10T12:35:15,511 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/C is initiating minor compaction (all files) 2024-12-10T12:35:15,511 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/C in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:15,511 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a1176b770d748349227cc9f673bea25, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1b29ff2d29e1473f8abe227ae1b2d2d4, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/306b221b571246f795bf689e732ec50b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/7f0db2c207d54b4ab27ba034d66e3f94] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=47.0 K 2024-12-10T12:35:15,511 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 9a1176b770d748349227cc9f673bea25, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733834108820 2024-12-10T12:35:15,512 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1b29ff2d29e1473f8abe227ae1b2d2d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733834109462 2024-12-10T12:35:15,512 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 306b221b571246f795bf689e732ec50b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733834110874 2024-12-10T12:35:15,512 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 7f0db2c207d54b4ab27ba034d66e3f94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733834113070 2024-12-10T12:35:15,515 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/aac8926214c14e0ea20688fb8891a29b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/aac8926214c14e0ea20688fb8891a29b 2024-12-10T12:35:15,522 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/A of 15f8962506199e1be1a11e7189a213ff into aac8926214c14e0ea20688fb8891a29b(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:15,522 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:15,522 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/A, priority=12, startTime=1733834115461; duration=0sec 2024-12-10T12:35:15,522 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:15,522 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:A 2024-12-10T12:35:15,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108f8b26632d7a40db95b52cf626541b56_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834114243/Put/seqid=0 2024-12-10T12:35:15,529 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#C#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:15,529 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/e2c3a02ea88b49fbb3df22fe4790ccc3 is 50, key is test_row_0/C:col10/1733834113070/Put/seqid=0 2024-12-10T12:35:15,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742179_1355 (size=12254) 2024-12-10T12:35:15,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742180_1356 (size=12241) 2024-12-10T12:35:15,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:15,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108f8b26632d7a40db95b52cf626541b56_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108f8b26632d7a40db95b52cf626541b56_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/8d2550faa7c74ab29f2a49d68014da65, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/8d2550faa7c74ab29f2a49d68014da65 is 175, key is test_row_0/A:col10/1733834114243/Put/seqid=0 2024-12-10T12:35:15,945 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/e2c3a02ea88b49fbb3df22fe4790ccc3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e2c3a02ea88b49fbb3df22fe4790ccc3 2024-12-10T12:35:15,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742181_1357 (size=31055) 2024-12-10T12:35:15,952 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/C of 15f8962506199e1be1a11e7189a213ff into e2c3a02ea88b49fbb3df22fe4790ccc3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:15,952 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:15,952 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/C, priority=12, startTime=1733834115462; duration=0sec 2024-12-10T12:35:15,952 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:15,952 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:C 2024-12-10T12:35:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-10T12:35:16,350 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/8d2550faa7c74ab29f2a49d68014da65 2024-12-10T12:35:16,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/5e57a246df6841d9824b9924c1f161d4 is 50, key is test_row_0/B:col10/1733834114243/Put/seqid=0 2024-12-10T12:35:16,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742182_1358 (size=12101) 2024-12-10T12:35:16,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834176437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:16,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834176541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:16,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834176744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:16,759 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/5e57a246df6841d9824b9924c1f161d4 2024-12-10T12:35:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c34f026d14b5447da53a932b51bb4ce0 is 50, key is test_row_0/C:col10/1733834114243/Put/seqid=0 2024-12-10T12:35:16,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742183_1359 (size=12101) 2024-12-10T12:35:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-10T12:35:17,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834177048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:17,190 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c34f026d14b5447da53a932b51bb4ce0 2024-12-10T12:35:17,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/8d2550faa7c74ab29f2a49d68014da65 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/8d2550faa7c74ab29f2a49d68014da65 2024-12-10T12:35:17,197 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/8d2550faa7c74ab29f2a49d68014da65, entries=150, sequenceid=132, filesize=30.3 K 2024-12-10T12:35:17,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/5e57a246df6841d9824b9924c1f161d4 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/5e57a246df6841d9824b9924c1f161d4 2024-12-10T12:35:17,200 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/5e57a246df6841d9824b9924c1f161d4, entries=150, sequenceid=132, filesize=11.8 K 2024-12-10T12:35:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c34f026d14b5447da53a932b51bb4ce0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c34f026d14b5447da53a932b51bb4ce0 2024-12-10T12:35:17,205 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c34f026d14b5447da53a932b51bb4ce0, entries=150, sequenceid=132, filesize=11.8 K 2024-12-10T12:35:17,205 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 15f8962506199e1be1a11e7189a213ff in 1711ms, sequenceid=132, compaction requested=false 2024-12-10T12:35:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:17,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=111 2024-12-10T12:35:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=111 2024-12-10T12:35:17,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=111, resume processing ppid=110 2024-12-10T12:35:17,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=111, ppid=110, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-12-10T12:35:17,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=110, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees in 2.3240 sec 2024-12-10T12:35:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:17,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-10T12:35:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:17,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109eaa590a71124320b06f105ee3094d63_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834116433/Put/seqid=0 2024-12-10T12:35:17,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742184_1360 (size=14794) 2024-12-10T12:35:17,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834177602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:17,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:17,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58770 deadline: 1733834177636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:17,640 DEBUG [Thread-1468 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8174 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:35:17,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58806 deadline: 1733834177639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:17,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58772 deadline: 1733834177640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:17,643 DEBUG [Thread-1470 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8176 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:35:17,643 DEBUG [Thread-1472 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8175 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:35:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58790 deadline: 1733834177648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:17,651 DEBUG [Thread-1474 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8185 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:35:17,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834177709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:17,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834177913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:17,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:17,970 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109eaa590a71124320b06f105ee3094d63_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109eaa590a71124320b06f105ee3094d63_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:17,970 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/db9ac313f0694e0fb29ceb4ad3f947d4, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:17,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/db9ac313f0694e0fb29ceb4ad3f947d4 is 175, key is test_row_0/A:col10/1733834116433/Put/seqid=0 2024-12-10T12:35:17,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742185_1361 (size=39749) 2024-12-10T12:35:17,974 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/db9ac313f0694e0fb29ceb4ad3f947d4 2024-12-10T12:35:17,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/42325a7a4b144cec8b15aee9ebf52844 is 50, key is test_row_0/B:col10/1733834116433/Put/seqid=0 2024-12-10T12:35:17,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742186_1362 (size=12151) 2024-12-10T12:35:18,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834178217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:18,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/42325a7a4b144cec8b15aee9ebf52844 2024-12-10T12:35:18,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/e600afc07d3e477fa8ae11517e8796d6 is 50, key is test_row_0/C:col10/1733834116433/Put/seqid=0 2024-12-10T12:35:18,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742187_1363 (size=12151) 2024-12-10T12:35:18,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834178720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:18,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/e600afc07d3e477fa8ae11517e8796d6 2024-12-10T12:35:18,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/db9ac313f0694e0fb29ceb4ad3f947d4 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db9ac313f0694e0fb29ceb4ad3f947d4 2024-12-10T12:35:18,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db9ac313f0694e0fb29ceb4ad3f947d4, entries=200, sequenceid=159, filesize=38.8 K 2024-12-10T12:35:18,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/42325a7a4b144cec8b15aee9ebf52844 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/42325a7a4b144cec8b15aee9ebf52844 2024-12-10T12:35:18,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/42325a7a4b144cec8b15aee9ebf52844, entries=150, sequenceid=159, filesize=11.9 K 2024-12-10T12:35:18,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/e600afc07d3e477fa8ae11517e8796d6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e600afc07d3e477fa8ae11517e8796d6 2024-12-10T12:35:18,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e600afc07d3e477fa8ae11517e8796d6, entries=150, sequenceid=159, filesize=11.9 K 2024-12-10T12:35:18,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 15f8962506199e1be1a11e7189a213ff in 1253ms, sequenceid=159, compaction requested=true 2024-12-10T12:35:18,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:18,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:18,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:18,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:18,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:18,809 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:18,809 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:18,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:18,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:18,810 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:18,810 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:18,811 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/A is initiating minor compaction (all files) 2024-12-10T12:35:18,811 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/B is initiating minor compaction (all files) 2024-12-10T12:35:18,811 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/A in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:18,811 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/B in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:18,811 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/aac8926214c14e0ea20688fb8891a29b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/8d2550faa7c74ab29f2a49d68014da65, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db9ac313f0694e0fb29ceb4ad3f947d4] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=99.6 K 2024-12-10T12:35:18,811 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/eff403eff245497091443ae4792455f7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/5e57a246df6841d9824b9924c1f161d4, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/42325a7a4b144cec8b15aee9ebf52844] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=35.6 K 2024-12-10T12:35:18,811 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:18,811 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/aac8926214c14e0ea20688fb8891a29b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/8d2550faa7c74ab29f2a49d68014da65, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db9ac313f0694e0fb29ceb4ad3f947d4] 2024-12-10T12:35:18,811 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting aac8926214c14e0ea20688fb8891a29b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733834113070 2024-12-10T12:35:18,811 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting eff403eff245497091443ae4792455f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733834113070 2024-12-10T12:35:18,811 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8d2550faa7c74ab29f2a49d68014da65, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733834114225 2024-12-10T12:35:18,811 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5e57a246df6841d9824b9924c1f161d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733834114225 2024-12-10T12:35:18,812 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting db9ac313f0694e0fb29ceb4ad3f947d4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733834116422 2024-12-10T12:35:18,812 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 42325a7a4b144cec8b15aee9ebf52844, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733834116422 2024-12-10T12:35:18,816 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:18,818 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:18,818 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/bd93d1c87dbb4c389fdf1b699ffe799e is 50, key is test_row_0/B:col10/1733834116433/Put/seqid=0 2024-12-10T12:35:18,827 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412104fce3bb6eeed44e1a61bf2c97535cb43_15f8962506199e1be1a11e7189a213ff store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:18,829 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412104fce3bb6eeed44e1a61bf2c97535cb43_15f8962506199e1be1a11e7189a213ff, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:18,829 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104fce3bb6eeed44e1a61bf2c97535cb43_15f8962506199e1be1a11e7189a213ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:18,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742188_1364 (size=12493) 2024-12-10T12:35:18,836 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/bd93d1c87dbb4c389fdf1b699ffe799e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/bd93d1c87dbb4c389fdf1b699ffe799e 2024-12-10T12:35:18,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742189_1365 (size=4469) 2024-12-10T12:35:18,841 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/B of 15f8962506199e1be1a11e7189a213ff into bd93d1c87dbb4c389fdf1b699ffe799e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:18,841 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:18,841 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/B, priority=13, startTime=1733834118809; duration=0sec 2024-12-10T12:35:18,841 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:18,841 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:B 2024-12-10T12:35:18,841 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:18,842 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:18,842 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/C is initiating minor compaction (all files) 2024-12-10T12:35:18,842 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/C in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:18,843 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e2c3a02ea88b49fbb3df22fe4790ccc3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c34f026d14b5447da53a932b51bb4ce0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e600afc07d3e477fa8ae11517e8796d6] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=35.6 K 2024-12-10T12:35:18,843 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting e2c3a02ea88b49fbb3df22fe4790ccc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733834113070 2024-12-10T12:35:18,843 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting c34f026d14b5447da53a932b51bb4ce0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733834114225 2024-12-10T12:35:18,844 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting e600afc07d3e477fa8ae11517e8796d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733834116422 2024-12-10T12:35:18,851 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:18,852 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/bd6c626fff924368b905e96c574e595c is 50, key is test_row_0/C:col10/1733834116433/Put/seqid=0 2024-12-10T12:35:18,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742190_1366 (size=12493) 2024-12-10T12:35:18,860 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/bd6c626fff924368b905e96c574e595c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/bd6c626fff924368b905e96c574e595c 2024-12-10T12:35:18,866 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/C of 15f8962506199e1be1a11e7189a213ff into bd6c626fff924368b905e96c574e595c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:18,866 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:18,866 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/C, priority=13, startTime=1733834118809; duration=0sec 2024-12-10T12:35:18,866 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:18,866 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:C 2024-12-10T12:35:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-10T12:35:18,989 INFO [Thread-1478 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 110 completed 2024-12-10T12:35:18,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=112, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=112, table=TestAcidGuarantees 2024-12-10T12:35:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-10T12:35:18,991 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=112, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=112, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:18,992 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=112, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=112, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:18,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-10T12:35:19,143 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:19,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=113 2024-12-10T12:35:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:19,143 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-10T12:35:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100bae5366d2df46f9a8d4026f58078553_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834117593/Put/seqid=0 2024-12-10T12:35:19,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742191_1367 (size=12304) 2024-12-10T12:35:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:19,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100bae5366d2df46f9a8d4026f58078553_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100bae5366d2df46f9a8d4026f58078553_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:19,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/f5be94cb6d834ad9a9b18139aec3def1, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:19,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/f5be94cb6d834ad9a9b18139aec3def1 is 175, key is test_row_0/A:col10/1733834117593/Put/seqid=0 2024-12-10T12:35:19,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742192_1368 (size=31105) 2024-12-10T12:35:19,165 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/f5be94cb6d834ad9a9b18139aec3def1 2024-12-10T12:35:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/06f74da60f64426a89416676397f11de is 50, key is test_row_0/B:col10/1733834117593/Put/seqid=0 2024-12-10T12:35:19,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742193_1369 (size=12151) 2024-12-10T12:35:19,238 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#A#compaction#306 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:19,238 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/d6da0380e5a947f4853a3ee467471154 is 175, key is test_row_0/A:col10/1733834116433/Put/seqid=0 2024-12-10T12:35:19,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742194_1370 (size=31447) 2024-12-10T12:35:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-10T12:35:19,590 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/06f74da60f64426a89416676397f11de 2024-12-10T12:35:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-10T12:35:19,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/1cb6953ac8fb4d1f8d3cc5c1bb43600e is 50, key is test_row_0/C:col10/1733834117593/Put/seqid=0 2024-12-10T12:35:19,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742195_1371 (size=12151) 2024-12-10T12:35:19,645 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/d6da0380e5a947f4853a3ee467471154 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d6da0380e5a947f4853a3ee467471154 2024-12-10T12:35:19,649 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/A of 15f8962506199e1be1a11e7189a213ff into d6da0380e5a947f4853a3ee467471154(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:19,649 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:19,649 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/A, priority=13, startTime=1733834118809; duration=0sec 2024-12-10T12:35:19,649 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:19,649 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:A 2024-12-10T12:35:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:19,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:19,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834179809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:19,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834179914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:20,002 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/1cb6953ac8fb4d1f8d3cc5c1bb43600e 2024-12-10T12:35:20,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/f5be94cb6d834ad9a9b18139aec3def1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f5be94cb6d834ad9a9b18139aec3def1 2024-12-10T12:35:20,009 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f5be94cb6d834ad9a9b18139aec3def1, entries=150, sequenceid=173, filesize=30.4 K 2024-12-10T12:35:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/06f74da60f64426a89416676397f11de as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06f74da60f64426a89416676397f11de 2024-12-10T12:35:20,014 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06f74da60f64426a89416676397f11de, entries=150, sequenceid=173, filesize=11.9 K 2024-12-10T12:35:20,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/1cb6953ac8fb4d1f8d3cc5c1bb43600e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1cb6953ac8fb4d1f8d3cc5c1bb43600e 2024-12-10T12:35:20,021 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1cb6953ac8fb4d1f8d3cc5c1bb43600e, entries=150, sequenceid=173, filesize=11.9 K 2024-12-10T12:35:20,022 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 15f8962506199e1be1a11e7189a213ff in 879ms, sequenceid=173, compaction requested=false 2024-12-10T12:35:20,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:20,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=113 2024-12-10T12:35:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=113 2024-12-10T12:35:20,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=113, resume processing ppid=112 2024-12-10T12:35:20,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=113, ppid=112, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0310 sec 2024-12-10T12:35:20,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=112, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=112, table=TestAcidGuarantees in 1.0350 sec 2024-12-10T12:35:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-10T12:35:20,094 INFO [Thread-1478 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 112 completed 2024-12-10T12:35:20,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=114, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=114, table=TestAcidGuarantees 2024-12-10T12:35:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-10T12:35:20,096 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=114, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=114, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:20,097 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=114, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=114, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:20,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:20,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:35:20,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:20,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:20,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:20,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:20,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:20,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:20,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210306a0077ee0e44778b49f9fbc2cdfd94_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834119800/Put/seqid=0 2024-12-10T12:35:20,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742196_1372 (size=14794) 2024-12-10T12:35:20,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834180146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-10T12:35:20,248 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:20,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=115 2024-12-10T12:35:20,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:20,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] handler.RSProcedureHandler(58): pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=115 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834180250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-10T12:35:20,400 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:20,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=115 2024-12-10T12:35:20,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:20,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] handler.RSProcedureHandler(58): pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=115 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834180454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:20,534 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:20,537 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210306a0077ee0e44778b49f9fbc2cdfd94_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210306a0077ee0e44778b49f9fbc2cdfd94_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:20,538 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/0fc6f8f3490d4e38b5ff0469cbcf84f0, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:20,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/0fc6f8f3490d4e38b5ff0469cbcf84f0 is 175, key is test_row_0/A:col10/1733834119800/Put/seqid=0 2024-12-10T12:35:20,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742197_1373 (size=39749) 2024-12-10T12:35:20,542 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/0fc6f8f3490d4e38b5ff0469cbcf84f0 2024-12-10T12:35:20,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/4fc7a474213541fdb8de13fce7581881 is 50, key is test_row_0/B:col10/1733834119800/Put/seqid=0 2024-12-10T12:35:20,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742198_1374 (size=12151) 2024-12-10T12:35:20,552 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:20,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=115 2024-12-10T12:35:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:20,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] handler.RSProcedureHandler(58): pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=115 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-10T12:35:20,704 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:20,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=115 2024-12-10T12:35:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] handler.RSProcedureHandler(58): pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=115 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834180759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:20,857 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:20,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=115 2024-12-10T12:35:20,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:20,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] handler.RSProcedureHandler(58): pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=115 java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=115 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:20,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/4fc7a474213541fdb8de13fce7581881 2024-12-10T12:35:20,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/2cf4ae63d2f94d449bc44213cfa50641 is 50, key is test_row_0/C:col10/1733834119800/Put/seqid=0 2024-12-10T12:35:20,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742199_1375 (size=12151) 2024-12-10T12:35:20,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/2cf4ae63d2f94d449bc44213cfa50641 2024-12-10T12:35:20,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/0fc6f8f3490d4e38b5ff0469cbcf84f0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/0fc6f8f3490d4e38b5ff0469cbcf84f0 2024-12-10T12:35:20,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/0fc6f8f3490d4e38b5ff0469cbcf84f0, entries=200, sequenceid=199, filesize=38.8 K 2024-12-10T12:35:20,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/4fc7a474213541fdb8de13fce7581881 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/4fc7a474213541fdb8de13fce7581881 2024-12-10T12:35:20,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/4fc7a474213541fdb8de13fce7581881, entries=150, sequenceid=199, filesize=11.9 K 2024-12-10T12:35:20,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/2cf4ae63d2f94d449bc44213cfa50641 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/2cf4ae63d2f94d449bc44213cfa50641 2024-12-10T12:35:20,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/2cf4ae63d2f94d449bc44213cfa50641, entries=150, sequenceid=199, filesize=11.9 K 2024-12-10T12:35:20,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 15f8962506199e1be1a11e7189a213ff in 855ms, sequenceid=199, compaction requested=true 2024-12-10T12:35:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:20,975 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:20,975 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:20,976 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:20,976 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:20,976 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/A is initiating minor compaction (all files) 2024-12-10T12:35:20,976 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/B is initiating minor compaction (all files) 2024-12-10T12:35:20,976 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/A in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,976 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/B in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,976 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d6da0380e5a947f4853a3ee467471154, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f5be94cb6d834ad9a9b18139aec3def1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/0fc6f8f3490d4e38b5ff0469cbcf84f0] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=99.9 K 2024-12-10T12:35:20,976 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/bd93d1c87dbb4c389fdf1b699ffe799e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06f74da60f64426a89416676397f11de, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/4fc7a474213541fdb8de13fce7581881] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=35.9 K 2024-12-10T12:35:20,976 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:20,976 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d6da0380e5a947f4853a3ee467471154, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f5be94cb6d834ad9a9b18139aec3def1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/0fc6f8f3490d4e38b5ff0469cbcf84f0] 2024-12-10T12:35:20,976 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting bd93d1c87dbb4c389fdf1b699ffe799e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733834116422 2024-12-10T12:35:20,976 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting d6da0380e5a947f4853a3ee467471154, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733834116422 2024-12-10T12:35:20,976 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 06f74da60f64426a89416676397f11de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733834117577 2024-12-10T12:35:20,976 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f5be94cb6d834ad9a9b18139aec3def1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733834117577 2024-12-10T12:35:20,977 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 4fc7a474213541fdb8de13fce7581881, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834119800 2024-12-10T12:35:20,977 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0fc6f8f3490d4e38b5ff0469cbcf84f0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834119800 2024-12-10T12:35:20,981 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:20,983 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:20,983 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412107386ea2e1b83428c8c1b04e2a9e08e81_15f8962506199e1be1a11e7189a213ff store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:20,983 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/441ec9af793244bfa4c0a17e76152fb7 is 50, key is test_row_0/B:col10/1733834119800/Put/seqid=0 2024-12-10T12:35:20,984 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412107386ea2e1b83428c8c1b04e2a9e08e81_15f8962506199e1be1a11e7189a213ff, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:20,984 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107386ea2e1b83428c8c1b04e2a9e08e81_15f8962506199e1be1a11e7189a213ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:20,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742200_1376 (size=12595) 2024-12-10T12:35:20,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742201_1377 (size=4469) 2024-12-10T12:35:20,992 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/441ec9af793244bfa4c0a17e76152fb7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/441ec9af793244bfa4c0a17e76152fb7 2024-12-10T12:35:20,992 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#A#compaction#315 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:20,992 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/696a55040d0d43f991ddb994716ffe93 is 175, key is test_row_0/A:col10/1733834119800/Put/seqid=0 2024-12-10T12:35:20,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742202_1378 (size=31549) 2024-12-10T12:35:20,998 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/B of 15f8962506199e1be1a11e7189a213ff into 441ec9af793244bfa4c0a17e76152fb7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:20,998 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:20,998 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/B, priority=13, startTime=1733834120975; duration=0sec 2024-12-10T12:35:20,998 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:20,998 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:B 2024-12-10T12:35:20,998 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:20,999 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:20,999 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/C is initiating minor compaction (all files) 2024-12-10T12:35:20,999 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/C in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:21,000 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/bd6c626fff924368b905e96c574e595c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1cb6953ac8fb4d1f8d3cc5c1bb43600e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/2cf4ae63d2f94d449bc44213cfa50641] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=35.9 K 2024-12-10T12:35:21,000 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting bd6c626fff924368b905e96c574e595c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733834116422 2024-12-10T12:35:21,000 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1cb6953ac8fb4d1f8d3cc5c1bb43600e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733834117577 2024-12-10T12:35:21,000 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2cf4ae63d2f94d449bc44213cfa50641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834119800 2024-12-10T12:35:21,006 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:21,006 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/e949c6f6df4c4810b3b979125e48dc00 is 50, key is test_row_0/C:col10/1733834119800/Put/seqid=0 2024-12-10T12:35:21,009 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:21,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742203_1379 (size=12595) 2024-12-10T12:35:21,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=115 2024-12-10T12:35:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:21,011 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:35:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:21,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121053e13a0240bb49dab651506f52f2e95c_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834120121/Put/seqid=0 2024-12-10T12:35:21,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742204_1380 (size=12304) 2024-12-10T12:35:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-10T12:35:21,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834181384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:21,400 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/696a55040d0d43f991ddb994716ffe93 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/696a55040d0d43f991ddb994716ffe93 2024-12-10T12:35:21,404 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/A of 15f8962506199e1be1a11e7189a213ff into 696a55040d0d43f991ddb994716ffe93(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:21,404 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:21,404 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/A, priority=13, startTime=1733834120975; duration=0sec 2024-12-10T12:35:21,404 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:21,404 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:A 2024-12-10T12:35:21,413 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/e949c6f6df4c4810b3b979125e48dc00 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e949c6f6df4c4810b3b979125e48dc00 2024-12-10T12:35:21,417 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/C of 15f8962506199e1be1a11e7189a213ff into e949c6f6df4c4810b3b979125e48dc00(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:21,417 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:21,417 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/C, priority=13, startTime=1733834120975; duration=0sec 2024-12-10T12:35:21,417 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:21,417 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:C 2024-12-10T12:35:21,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:21,425 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121053e13a0240bb49dab651506f52f2e95c_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121053e13a0240bb49dab651506f52f2e95c_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:21,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/7d440cad1dd64eb380ee4b086a2cd8da, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:21,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/7d440cad1dd64eb380ee4b086a2cd8da is 175, key is test_row_0/A:col10/1733834120121/Put/seqid=0 2024-12-10T12:35:21,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742205_1381 (size=31105) 2024-12-10T12:35:21,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834181489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:21,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834181691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:21,839 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/7d440cad1dd64eb380ee4b086a2cd8da 2024-12-10T12:35:21,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/122c35508b2b43efacadc6e7c6cd2329 is 50, key is test_row_0/B:col10/1733834120121/Put/seqid=0 2024-12-10T12:35:21,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742206_1382 (size=12151) 2024-12-10T12:35:21,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834181994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-10T12:35:22,274 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/122c35508b2b43efacadc6e7c6cd2329 2024-12-10T12:35:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/19d37d2bbb624945bb326a5fb2dfa5b0 is 50, key is test_row_0/C:col10/1733834120121/Put/seqid=0 2024-12-10T12:35:22,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742207_1383 (size=12151) 2024-12-10T12:35:22,287 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/19d37d2bbb624945bb326a5fb2dfa5b0 2024-12-10T12:35:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/7d440cad1dd64eb380ee4b086a2cd8da as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7d440cad1dd64eb380ee4b086a2cd8da 2024-12-10T12:35:22,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7d440cad1dd64eb380ee4b086a2cd8da, entries=150, sequenceid=211, filesize=30.4 K 2024-12-10T12:35:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/122c35508b2b43efacadc6e7c6cd2329 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/122c35508b2b43efacadc6e7c6cd2329 2024-12-10T12:35:22,298 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/122c35508b2b43efacadc6e7c6cd2329, entries=150, sequenceid=211, filesize=11.9 K 2024-12-10T12:35:22,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/19d37d2bbb624945bb326a5fb2dfa5b0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/19d37d2bbb624945bb326a5fb2dfa5b0 2024-12-10T12:35:22,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/19d37d2bbb624945bb326a5fb2dfa5b0, entries=150, sequenceid=211, filesize=11.9 K 2024-12-10T12:35:22,302 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 15f8962506199e1be1a11e7189a213ff in 1291ms, sequenceid=211, compaction requested=false 2024-12-10T12:35:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-12-10T12:35:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=115 2024-12-10T12:35:22,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=115, resume processing ppid=114 2024-12-10T12:35:22,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=115, ppid=114, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2060 sec 2024-12-10T12:35:22,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=114, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=114, table=TestAcidGuarantees in 2.2090 sec 2024-12-10T12:35:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:22,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-10T12:35:22,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:22,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:22,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:22,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:22,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:22,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:22,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102ecbe7126d274c3faa2119331d82cc53_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834121376/Put/seqid=0 2024-12-10T12:35:22,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742208_1384 (size=14794) 2024-12-10T12:35:22,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:22,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102ecbe7126d274c3faa2119331d82cc53_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102ecbe7126d274c3faa2119331d82cc53_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:22,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c7ee36a6ddf748da8dd5a706ebe01d4b, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:22,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c7ee36a6ddf748da8dd5a706ebe01d4b is 175, key is test_row_0/A:col10/1733834121376/Put/seqid=0 2024-12-10T12:35:22,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742209_1385 (size=39749) 2024-12-10T12:35:22,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834182532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:22,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834182636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:22,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834182839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:22,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c7ee36a6ddf748da8dd5a706ebe01d4b 2024-12-10T12:35:22,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/70cced6753024203b994f014d964c0e0 is 50, key is test_row_0/B:col10/1733834121376/Put/seqid=0 2024-12-10T12:35:22,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742210_1386 (size=12151) 2024-12-10T12:35:23,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834183143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:23,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/70cced6753024203b994f014d964c0e0 2024-12-10T12:35:23,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/76f42e3fc3c541cb9e90ff2f0ff34d9b is 50, key is test_row_0/C:col10/1733834121376/Put/seqid=0 2024-12-10T12:35:23,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742211_1387 (size=12151) 2024-12-10T12:35:23,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834183650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:23,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/76f42e3fc3c541cb9e90ff2f0ff34d9b 2024-12-10T12:35:23,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c7ee36a6ddf748da8dd5a706ebe01d4b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c7ee36a6ddf748da8dd5a706ebe01d4b 2024-12-10T12:35:23,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c7ee36a6ddf748da8dd5a706ebe01d4b, entries=200, sequenceid=239, filesize=38.8 K 2024-12-10T12:35:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/70cced6753024203b994f014d964c0e0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/70cced6753024203b994f014d964c0e0 2024-12-10T12:35:23,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/70cced6753024203b994f014d964c0e0, entries=150, sequenceid=239, filesize=11.9 K 2024-12-10T12:35:23,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/76f42e3fc3c541cb9e90ff2f0ff34d9b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/76f42e3fc3c541cb9e90ff2f0ff34d9b 2024-12-10T12:35:23,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/76f42e3fc3c541cb9e90ff2f0ff34d9b, entries=150, sequenceid=239, filesize=11.9 K 2024-12-10T12:35:23,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 15f8962506199e1be1a11e7189a213ff in 1258ms, sequenceid=239, compaction requested=true 2024-12-10T12:35:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:23,761 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:23,761 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15f8962506199e1be1a11e7189a213ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:23,762 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:23,762 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:23,762 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/B is initiating minor compaction (all files) 2024-12-10T12:35:23,762 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/A is initiating minor compaction (all files) 2024-12-10T12:35:23,762 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/B in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:23,762 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/A in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:23,762 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/696a55040d0d43f991ddb994716ffe93, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7d440cad1dd64eb380ee4b086a2cd8da, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c7ee36a6ddf748da8dd5a706ebe01d4b] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=100.0 K 2024-12-10T12:35:23,762 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/441ec9af793244bfa4c0a17e76152fb7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/122c35508b2b43efacadc6e7c6cd2329, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/70cced6753024203b994f014d964c0e0] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=36.0 K 2024-12-10T12:35:23,762 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:23,763 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/696a55040d0d43f991ddb994716ffe93, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7d440cad1dd64eb380ee4b086a2cd8da, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c7ee36a6ddf748da8dd5a706ebe01d4b] 2024-12-10T12:35:23,763 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 696a55040d0d43f991ddb994716ffe93, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834119800 2024-12-10T12:35:23,763 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 441ec9af793244bfa4c0a17e76152fb7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834119800 2024-12-10T12:35:23,763 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7d440cad1dd64eb380ee4b086a2cd8da, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733834120121 2024-12-10T12:35:23,763 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 122c35508b2b43efacadc6e7c6cd2329, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733834120121 2024-12-10T12:35:23,763 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting c7ee36a6ddf748da8dd5a706ebe01d4b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733834121359 2024-12-10T12:35:23,763 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 70cced6753024203b994f014d964c0e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733834121376 2024-12-10T12:35:23,769 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:23,769 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/88123981144445a0bd917555918ec0c6 is 50, key is test_row_0/B:col10/1733834121376/Put/seqid=0 2024-12-10T12:35:23,770 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:23,772 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210bff3ed87469b4b9f8bd4d022ceb01fb6_15f8962506199e1be1a11e7189a213ff store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:23,774 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210bff3ed87469b4b9f8bd4d022ceb01fb6_15f8962506199e1be1a11e7189a213ff, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:23,774 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bff3ed87469b4b9f8bd4d022ceb01fb6_15f8962506199e1be1a11e7189a213ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:23,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742212_1388 (size=12697) 2024-12-10T12:35:23,780 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/88123981144445a0bd917555918ec0c6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/88123981144445a0bd917555918ec0c6 2024-12-10T12:35:23,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742213_1389 (size=4469) 2024-12-10T12:35:23,782 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#A#compaction#325 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:23,783 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c4940c3752e040f3b3169de34ee0714e is 175, key is test_row_0/A:col10/1733834121376/Put/seqid=0 2024-12-10T12:35:23,784 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/B of 15f8962506199e1be1a11e7189a213ff into 88123981144445a0bd917555918ec0c6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:23,785 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:23,785 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/B, priority=13, startTime=1733834123761; duration=0sec 2024-12-10T12:35:23,785 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:23,785 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:B 2024-12-10T12:35:23,785 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:23,786 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:23,786 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 15f8962506199e1be1a11e7189a213ff/C is initiating minor compaction (all files) 2024-12-10T12:35:23,786 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 15f8962506199e1be1a11e7189a213ff/C in TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:23,786 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e949c6f6df4c4810b3b979125e48dc00, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/19d37d2bbb624945bb326a5fb2dfa5b0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/76f42e3fc3c541cb9e90ff2f0ff34d9b] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp, totalSize=36.0 K 2024-12-10T12:35:23,787 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting e949c6f6df4c4810b3b979125e48dc00, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733834119800 2024-12-10T12:35:23,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742214_1390 (size=31651) 2024-12-10T12:35:23,787 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 19d37d2bbb624945bb326a5fb2dfa5b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733834120121 2024-12-10T12:35:23,787 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 76f42e3fc3c541cb9e90ff2f0ff34d9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733834121376 2024-12-10T12:35:23,795 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15f8962506199e1be1a11e7189a213ff#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:23,796 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c7c7999a98be43e496757c8e486c7cfa is 50, key is test_row_0/C:col10/1733834121376/Put/seqid=0 2024-12-10T12:35:23,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742215_1391 (size=12697) 2024-12-10T12:35:23,802 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c7c7999a98be43e496757c8e486c7cfa as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c7c7999a98be43e496757c8e486c7cfa 2024-12-10T12:35:23,806 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/C of 15f8962506199e1be1a11e7189a213ff into c7c7999a98be43e496757c8e486c7cfa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:23,807 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:23,807 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/C, priority=13, startTime=1733834123761; duration=0sec 2024-12-10T12:35:23,807 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:23,807 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:C 2024-12-10T12:35:24,190 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c4940c3752e040f3b3169de34ee0714e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c4940c3752e040f3b3169de34ee0714e 2024-12-10T12:35:24,193 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 15f8962506199e1be1a11e7189a213ff/A of 15f8962506199e1be1a11e7189a213ff into c4940c3752e040f3b3169de34ee0714e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:24,193 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:24,193 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff., storeName=15f8962506199e1be1a11e7189a213ff/A, priority=13, startTime=1733834123761; duration=0sec 2024-12-10T12:35:24,194 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:24,194 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15f8962506199e1be1a11e7189a213ff:A 2024-12-10T12:35:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-10T12:35:24,201 INFO [Thread-1478 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 114 completed 2024-12-10T12:35:24,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=116, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=116, table=TestAcidGuarantees 2024-12-10T12:35:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-10T12:35:24,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=116, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=116, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:24,203 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=116, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=116, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:24,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-10T12:35:24,354 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:24,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=117 2024-12-10T12:35:24,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:24,354 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-10T12:35:24,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:24,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:24,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:24,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:24,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:24,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cc50331479bd4ff791a63a6ab39538f2_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834122504/Put/seqid=0 2024-12-10T12:35:24,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742216_1392 (size=12304) 2024-12-10T12:35:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-10T12:35:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:24,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:24,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834184731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:24,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:24,766 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cc50331479bd4ff791a63a6ab39538f2_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cc50331479bd4ff791a63a6ab39538f2_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:24,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c305ddf3a6394f1abd2dd8fa63a7792e, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c305ddf3a6394f1abd2dd8fa63a7792e is 175, key is test_row_0/A:col10/1733834122504/Put/seqid=0 2024-12-10T12:35:24,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742217_1393 (size=31105) 2024-12-10T12:35:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-10T12:35:24,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834184835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:25,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834185040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:25,171 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c305ddf3a6394f1abd2dd8fa63a7792e 2024-12-10T12:35:25,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/532eca5fd3634fe084caa8db24a5d647 is 50, key is test_row_0/B:col10/1733834122504/Put/seqid=0 2024-12-10T12:35:25,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742218_1394 (size=12151) 2024-12-10T12:35:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-10T12:35:25,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834185343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:25,581 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/532eca5fd3634fe084caa8db24a5d647 2024-12-10T12:35:25,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c2cab22af82f41bfbd87e6180c14cf1e is 50, key is test_row_0/C:col10/1733834122504/Put/seqid=0 2024-12-10T12:35:25,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742219_1395 (size=12151) 2024-12-10T12:35:25,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58798 deadline: 1733834185849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 2024-12-10T12:35:25,990 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c2cab22af82f41bfbd87e6180c14cf1e 2024-12-10T12:35:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/c305ddf3a6394f1abd2dd8fa63a7792e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c305ddf3a6394f1abd2dd8fa63a7792e 2024-12-10T12:35:25,996 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c305ddf3a6394f1abd2dd8fa63a7792e, entries=150, sequenceid=252, filesize=30.4 K 2024-12-10T12:35:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/532eca5fd3634fe084caa8db24a5d647 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/532eca5fd3634fe084caa8db24a5d647 2024-12-10T12:35:26,000 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/532eca5fd3634fe084caa8db24a5d647, entries=150, sequenceid=252, filesize=11.9 K 2024-12-10T12:35:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/c2cab22af82f41bfbd87e6180c14cf1e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c2cab22af82f41bfbd87e6180c14cf1e 2024-12-10T12:35:26,003 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c2cab22af82f41bfbd87e6180c14cf1e, entries=150, sequenceid=252, filesize=11.9 K 2024-12-10T12:35:26,003 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HRegion(3040): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 15f8962506199e1be1a11e7189a213ff in 1649ms, sequenceid=252, compaction requested=false 2024-12-10T12:35:26,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:26,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:26,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=117}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=117 2024-12-10T12:35:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=117 2024-12-10T12:35:26,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=117, resume processing ppid=116 2024-12-10T12:35:26,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=117, ppid=116, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8010 sec 2024-12-10T12:35:26,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=116, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=116, table=TestAcidGuarantees in 1.8040 sec 2024-12-10T12:35:26,112 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T12:35:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-10T12:35:26,306 INFO [Thread-1478 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 116 completed 2024-12-10T12:35:26,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=118, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=118, table=TestAcidGuarantees 2024-12-10T12:35:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-10T12:35:26,308 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=118, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=118, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:26,308 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=118, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=118, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:26,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-10T12:35:26,459 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:26,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=119 2024-12-10T12:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:26,460 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-10T12:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:26,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109dc9d7162148498481f99475fff416c8_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834124700/Put/seqid=0 2024-12-10T12:35:26,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742220_1396 (size=12454) 2024-12-10T12:35:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-10T12:35:26,668 DEBUG [Thread-1479 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5e3203d9 to 127.0.0.1:64063 2024-12-10T12:35:26,668 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:26,669 DEBUG [Thread-1485 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x37a637ac to 127.0.0.1:64063 2024-12-10T12:35:26,669 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:26,671 DEBUG [Thread-1487 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3fa53591 to 127.0.0.1:64063 2024-12-10T12:35:26,671 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:26,671 DEBUG [Thread-1481 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x798e7fd4 to 127.0.0.1:64063 2024-12-10T12:35:26,671 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:26,671 DEBUG [Thread-1483 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7284f16d to 127.0.0.1:64063 2024-12-10T12:35:26,671 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:26,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. as already flushing 2024-12-10T12:35:26,858 DEBUG [Thread-1476 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2205f666 to 127.0.0.1:64063 2024-12-10T12:35:26,859 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:26,877 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109dc9d7162148498481f99475fff416c8_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109dc9d7162148498481f99475fff416c8_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:26,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/6d0eaff45d504243bfeaa66573650c43, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/6d0eaff45d504243bfeaa66573650c43 is 175, key is test_row_0/A:col10/1733834124700/Put/seqid=0 2024-12-10T12:35:26,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742221_1397 (size=31255) 2024-12-10T12:35:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-10T12:35:27,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/6d0eaff45d504243bfeaa66573650c43 2024-12-10T12:35:27,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/e76ec2f680d44376ad039829b88fc76d is 50, key is test_row_0/B:col10/1733834124700/Put/seqid=0 2024-12-10T12:35:27,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742222_1398 (size=12301) 2024-12-10T12:35:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-10T12:35:27,656 DEBUG [Thread-1472 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31aea41b to 127.0.0.1:64063 2024-12-10T12:35:27,656 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:27,672 DEBUG [Thread-1474 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1e247aa1 to 127.0.0.1:64063 2024-12-10T12:35:27,672 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:27,684 DEBUG [Thread-1468 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x117e86d9 to 127.0.0.1:64063 2024-12-10T12:35:27,684 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:27,703 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/e76ec2f680d44376ad039829b88fc76d 2024-12-10T12:35:27,711 DEBUG [Thread-1470 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6cd96549 to 127.0.0.1:64063 2024-12-10T12:35:27,712 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:27,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/07f018412f3144ada2065f147ca7ad4b is 50, key is test_row_0/C:col10/1733834124700/Put/seqid=0 2024-12-10T12:35:27,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742223_1399 (size=12301) 2024-12-10T12:35:28,121 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/07f018412f3144ada2065f147ca7ad4b 2024-12-10T12:35:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/6d0eaff45d504243bfeaa66573650c43 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/6d0eaff45d504243bfeaa66573650c43 2024-12-10T12:35:28,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/6d0eaff45d504243bfeaa66573650c43, entries=150, sequenceid=278, filesize=30.5 K 2024-12-10T12:35:28,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/e76ec2f680d44376ad039829b88fc76d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/e76ec2f680d44376ad039829b88fc76d 2024-12-10T12:35:28,142 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/e76ec2f680d44376ad039829b88fc76d, entries=150, sequenceid=278, filesize=12.0 K 2024-12-10T12:35:28,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/07f018412f3144ada2065f147ca7ad4b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/07f018412f3144ada2065f147ca7ad4b 2024-12-10T12:35:28,146 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/07f018412f3144ada2065f147ca7ad4b, entries=150, sequenceid=278, filesize=12.0 K 2024-12-10T12:35:28,147 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=33.54 KB/34350 for 15f8962506199e1be1a11e7189a213ff in 1687ms, sequenceid=278, compaction requested=true 2024-12-10T12:35:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegion(2538): Flush status journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=119 2024-12-10T12:35:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=119 2024-12-10T12:35:28,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=119, resume processing ppid=118 2024-12-10T12:35:28,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=119, ppid=118, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-12-10T12:35:28,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=118, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=118, table=TestAcidGuarantees in 1.8430 sec 2024-12-10T12:35:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-10T12:35:28,413 INFO [Thread-1478 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 118 completed 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 163 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3529 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10587 rows 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3510 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10529 rows 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3511 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10533 rows 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3510 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10529 rows 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3526 2024-12-10T12:35:28,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10578 rows 2024-12-10T12:35:28,413 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-10T12:35:28,413 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1dc42ea6 to 127.0.0.1:64063 2024-12-10T12:35:28,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:28,414 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-12-10T12:35:28,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T12:35:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=120, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-12-10T12:35:28,419 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834128418"}]},"ts":"1733834128418"} 2024-12-10T12:35:28,420 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T12:35:28,464 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T12:35:28,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T12:35:28,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, UNASSIGN}] 2024-12-10T12:35:28,468 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, UNASSIGN 2024-12-10T12:35:28,469 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=15f8962506199e1be1a11e7189a213ff, regionState=CLOSING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:28,471 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T12:35:28,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE; CloseRegionProcedure 15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-12-10T12:35:28,623 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:28,624 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(124): Close 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:28,624 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-12-10T12:35:28,624 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1681): Closing 15f8962506199e1be1a11e7189a213ff, disabling compactions & flushes 2024-12-10T12:35:28,625 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:28,625 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:28,625 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. after waiting 0 ms 2024-12-10T12:35:28,625 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:28,625 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(2837): Flushing 15f8962506199e1be1a11e7189a213ff 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-10T12:35:28,625 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=A 2024-12-10T12:35:28,626 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:28,626 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=B 2024-12-10T12:35:28,626 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:28,626 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 15f8962506199e1be1a11e7189a213ff, store=C 2024-12-10T12:35:28,626 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:28,635 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109d84036311c542c3a479c39a90f70fb5_15f8962506199e1be1a11e7189a213ff is 50, key is test_row_0/A:col10/1733834126855/Put/seqid=0 2024-12-10T12:35:28,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742224_1400 (size=12454) 2024-12-10T12:35:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-12-10T12:35:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-12-10T12:35:29,042 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:29,052 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109d84036311c542c3a479c39a90f70fb5_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109d84036311c542c3a479c39a90f70fb5_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:29,054 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/db65bb2f736c4925aeb9ea72ed5a0824, store: [table=TestAcidGuarantees family=A region=15f8962506199e1be1a11e7189a213ff] 2024-12-10T12:35:29,055 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/db65bb2f736c4925aeb9ea72ed5a0824 is 175, key is test_row_0/A:col10/1733834126855/Put/seqid=0 2024-12-10T12:35:29,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742225_1401 (size=31255) 2024-12-10T12:35:29,461 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/db65bb2f736c4925aeb9ea72ed5a0824 2024-12-10T12:35:29,476 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/10e3705c24cd44d4b5990c22badfb644 is 50, key is test_row_0/B:col10/1733834126855/Put/seqid=0 2024-12-10T12:35:29,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742226_1402 (size=12301) 2024-12-10T12:35:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-12-10T12:35:29,880 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/10e3705c24cd44d4b5990c22badfb644 2024-12-10T12:35:29,885 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/402d118ab6f04ebbbce541081ecfe740 is 50, key is test_row_0/C:col10/1733834126855/Put/seqid=0 2024-12-10T12:35:29,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742227_1403 (size=12301) 2024-12-10T12:35:29,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fad905551524:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7216ffd238094eb79a47879a3e19bdba, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f90e565ab1a24a7f89a0372ecb51e0de, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/4a3439ad9c0c4f6884766ad2e5632dc5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/380b5e7653d64d0598a3847322d06dec, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5230445b40574f2f8916d6d33da34de7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5b1ec42b93e2433c9565be6d9fbbff45, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d80c0cba52564b0bbf365cb271170979, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/aac8926214c14e0ea20688fb8891a29b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/8d2550faa7c74ab29f2a49d68014da65, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db9ac313f0694e0fb29ceb4ad3f947d4, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d6da0380e5a947f4853a3ee467471154, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f5be94cb6d834ad9a9b18139aec3def1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/0fc6f8f3490d4e38b5ff0469cbcf84f0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/696a55040d0d43f991ddb994716ffe93, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7d440cad1dd64eb380ee4b086a2cd8da, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c7ee36a6ddf748da8dd5a706ebe01d4b] to archive 2024-12-10T12:35:29,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fad905551524:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:35:29,967 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7216ffd238094eb79a47879a3e19bdba to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7216ffd238094eb79a47879a3e19bdba 2024-12-10T12:35:29,967 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/4a3439ad9c0c4f6884766ad2e5632dc5 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/4a3439ad9c0c4f6884766ad2e5632dc5 2024-12-10T12:35:29,967 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5230445b40574f2f8916d6d33da34de7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5230445b40574f2f8916d6d33da34de7 2024-12-10T12:35:29,967 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5b1ec42b93e2433c9565be6d9fbbff45 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/5b1ec42b93e2433c9565be6d9fbbff45 2024-12-10T12:35:29,967 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f90e565ab1a24a7f89a0372ecb51e0de to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f90e565ab1a24a7f89a0372ecb51e0de 2024-12-10T12:35:29,967 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d80c0cba52564b0bbf365cb271170979 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d80c0cba52564b0bbf365cb271170979 2024-12-10T12:35:29,967 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/aac8926214c14e0ea20688fb8891a29b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/aac8926214c14e0ea20688fb8891a29b 2024-12-10T12:35:29,967 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/380b5e7653d64d0598a3847322d06dec to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/380b5e7653d64d0598a3847322d06dec 2024-12-10T12:35:29,968 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/8d2550faa7c74ab29f2a49d68014da65 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/8d2550faa7c74ab29f2a49d68014da65 2024-12-10T12:35:29,968 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d6da0380e5a947f4853a3ee467471154 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/d6da0380e5a947f4853a3ee467471154 2024-12-10T12:35:29,968 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db9ac313f0694e0fb29ceb4ad3f947d4 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db9ac313f0694e0fb29ceb4ad3f947d4 2024-12-10T12:35:29,968 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/696a55040d0d43f991ddb994716ffe93 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/696a55040d0d43f991ddb994716ffe93 2024-12-10T12:35:29,969 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f5be94cb6d834ad9a9b18139aec3def1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/f5be94cb6d834ad9a9b18139aec3def1 2024-12-10T12:35:29,969 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c7ee36a6ddf748da8dd5a706ebe01d4b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c7ee36a6ddf748da8dd5a706ebe01d4b 2024-12-10T12:35:29,969 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/0fc6f8f3490d4e38b5ff0469cbcf84f0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/0fc6f8f3490d4e38b5ff0469cbcf84f0 2024-12-10T12:35:29,969 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7d440cad1dd64eb380ee4b086a2cd8da to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/7d440cad1dd64eb380ee4b086a2cd8da 2024-12-10T12:35:29,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fad905551524:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06b6e2ac86974e9a949bbe567c5f1693, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/602f0bfe5aa34945a53a9b63e3341633, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/f318fc93fe1241fe9d0f5e8a346d126d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/1ea07ba3f81c4042a0f68e9cdc3bba62, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/fca701b5ac7842f6a6b62c3a2c2bf78a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/86176f03bf644f51ac090f53c0544a05, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/eff403eff245497091443ae4792455f7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/b880da6d2f704b2e8e51265fc3f2868f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/5e57a246df6841d9824b9924c1f161d4, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/bd93d1c87dbb4c389fdf1b699ffe799e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/42325a7a4b144cec8b15aee9ebf52844, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06f74da60f64426a89416676397f11de, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/441ec9af793244bfa4c0a17e76152fb7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/4fc7a474213541fdb8de13fce7581881, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/122c35508b2b43efacadc6e7c6cd2329, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/70cced6753024203b994f014d964c0e0] to archive 2024-12-10T12:35:29,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fad905551524:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:35:29,973 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06b6e2ac86974e9a949bbe567c5f1693 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06b6e2ac86974e9a949bbe567c5f1693 2024-12-10T12:35:29,973 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/602f0bfe5aa34945a53a9b63e3341633 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/602f0bfe5aa34945a53a9b63e3341633 2024-12-10T12:35:29,973 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/f318fc93fe1241fe9d0f5e8a346d126d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/f318fc93fe1241fe9d0f5e8a346d126d 2024-12-10T12:35:29,973 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/1ea07ba3f81c4042a0f68e9cdc3bba62 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/1ea07ba3f81c4042a0f68e9cdc3bba62 2024-12-10T12:35:29,973 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/86176f03bf644f51ac090f53c0544a05 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/86176f03bf644f51ac090f53c0544a05 2024-12-10T12:35:29,973 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/b880da6d2f704b2e8e51265fc3f2868f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/b880da6d2f704b2e8e51265fc3f2868f 2024-12-10T12:35:29,973 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/fca701b5ac7842f6a6b62c3a2c2bf78a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/fca701b5ac7842f6a6b62c3a2c2bf78a 2024-12-10T12:35:29,974 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/eff403eff245497091443ae4792455f7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/eff403eff245497091443ae4792455f7 2024-12-10T12:35:29,974 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/42325a7a4b144cec8b15aee9ebf52844 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/42325a7a4b144cec8b15aee9ebf52844 2024-12-10T12:35:29,974 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/5e57a246df6841d9824b9924c1f161d4 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/5e57a246df6841d9824b9924c1f161d4 2024-12-10T12:35:29,974 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06f74da60f64426a89416676397f11de to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/06f74da60f64426a89416676397f11de 2024-12-10T12:35:29,974 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/441ec9af793244bfa4c0a17e76152fb7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/441ec9af793244bfa4c0a17e76152fb7 2024-12-10T12:35:29,974 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/bd93d1c87dbb4c389fdf1b699ffe799e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/bd93d1c87dbb4c389fdf1b699ffe799e 2024-12-10T12:35:29,975 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/4fc7a474213541fdb8de13fce7581881 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/4fc7a474213541fdb8de13fce7581881 2024-12-10T12:35:29,975 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/122c35508b2b43efacadc6e7c6cd2329 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/122c35508b2b43efacadc6e7c6cd2329 2024-12-10T12:35:29,975 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/70cced6753024203b994f014d964c0e0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/70cced6753024203b994f014d964c0e0 2024-12-10T12:35:29,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fad905551524:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a41c89aaa284f208fc3c07db5ad1712, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/4c81c9ebeb7d4fd1acdf00c9cb6f67a7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a1176b770d748349227cc9f673bea25, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c7c80e486a864a1f8b8d296be794d5f0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1b29ff2d29e1473f8abe227ae1b2d2d4, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/306b221b571246f795bf689e732ec50b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e2c3a02ea88b49fbb3df22fe4790ccc3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/7f0db2c207d54b4ab27ba034d66e3f94, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c34f026d14b5447da53a932b51bb4ce0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/bd6c626fff924368b905e96c574e595c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e600afc07d3e477fa8ae11517e8796d6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1cb6953ac8fb4d1f8d3cc5c1bb43600e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e949c6f6df4c4810b3b979125e48dc00, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/2cf4ae63d2f94d449bc44213cfa50641, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/19d37d2bbb624945bb326a5fb2dfa5b0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/76f42e3fc3c541cb9e90ff2f0ff34d9b] to archive 2024-12-10T12:35:29,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fad905551524:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:35:29,978 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a41c89aaa284f208fc3c07db5ad1712 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a41c89aaa284f208fc3c07db5ad1712 2024-12-10T12:35:29,979 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/4c81c9ebeb7d4fd1acdf00c9cb6f67a7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/4c81c9ebeb7d4fd1acdf00c9cb6f67a7 2024-12-10T12:35:29,979 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c7c80e486a864a1f8b8d296be794d5f0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c7c80e486a864a1f8b8d296be794d5f0 2024-12-10T12:35:29,979 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1b29ff2d29e1473f8abe227ae1b2d2d4 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1b29ff2d29e1473f8abe227ae1b2d2d4 2024-12-10T12:35:29,979 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e2c3a02ea88b49fbb3df22fe4790ccc3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e2c3a02ea88b49fbb3df22fe4790ccc3 2024-12-10T12:35:29,979 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/306b221b571246f795bf689e732ec50b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/306b221b571246f795bf689e732ec50b 2024-12-10T12:35:29,979 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a1176b770d748349227cc9f673bea25 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/9a1176b770d748349227cc9f673bea25 2024-12-10T12:35:29,979 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/7f0db2c207d54b4ab27ba034d66e3f94 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/7f0db2c207d54b4ab27ba034d66e3f94 2024-12-10T12:35:29,980 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c34f026d14b5447da53a932b51bb4ce0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c34f026d14b5447da53a932b51bb4ce0 2024-12-10T12:35:29,980 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1cb6953ac8fb4d1f8d3cc5c1bb43600e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/1cb6953ac8fb4d1f8d3cc5c1bb43600e 2024-12-10T12:35:29,980 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/2cf4ae63d2f94d449bc44213cfa50641 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/2cf4ae63d2f94d449bc44213cfa50641 2024-12-10T12:35:29,980 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/bd6c626fff924368b905e96c574e595c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/bd6c626fff924368b905e96c574e595c 2024-12-10T12:35:29,980 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e600afc07d3e477fa8ae11517e8796d6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e600afc07d3e477fa8ae11517e8796d6 2024-12-10T12:35:29,980 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e949c6f6df4c4810b3b979125e48dc00 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/e949c6f6df4c4810b3b979125e48dc00 2024-12-10T12:35:29,981 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/19d37d2bbb624945bb326a5fb2dfa5b0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/19d37d2bbb624945bb326a5fb2dfa5b0 2024-12-10T12:35:29,981 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/76f42e3fc3c541cb9e90ff2f0ff34d9b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/76f42e3fc3c541cb9e90ff2f0ff34d9b 2024-12-10T12:35:30,290 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/402d118ab6f04ebbbce541081ecfe740 2024-12-10T12:35:30,299 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/A/db65bb2f736c4925aeb9ea72ed5a0824 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db65bb2f736c4925aeb9ea72ed5a0824 2024-12-10T12:35:30,305 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db65bb2f736c4925aeb9ea72ed5a0824, entries=150, sequenceid=286, filesize=30.5 K 2024-12-10T12:35:30,306 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/B/10e3705c24cd44d4b5990c22badfb644 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/10e3705c24cd44d4b5990c22badfb644 2024-12-10T12:35:30,309 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/10e3705c24cd44d4b5990c22badfb644, entries=150, sequenceid=286, filesize=12.0 K 2024-12-10T12:35:30,309 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/.tmp/C/402d118ab6f04ebbbce541081ecfe740 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/402d118ab6f04ebbbce541081ecfe740 2024-12-10T12:35:30,312 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/402d118ab6f04ebbbce541081ecfe740, entries=150, sequenceid=286, filesize=12.0 K 2024-12-10T12:35:30,313 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(3040): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 15f8962506199e1be1a11e7189a213ff in 1688ms, sequenceid=286, compaction requested=true 2024-12-10T12:35:30,317 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/recovered.edits/289.seqid, newMaxSeqId=289, maxSeqId=4 2024-12-10T12:35:30,317 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff. 2024-12-10T12:35:30,317 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1635): Region close journal for 15f8962506199e1be1a11e7189a213ff: 2024-12-10T12:35:30,319 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(170): Closed 15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,319 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=15f8962506199e1be1a11e7189a213ff, regionState=CLOSED 2024-12-10T12:35:30,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=123, resume processing ppid=122 2024-12-10T12:35:30,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=123, ppid=122, state=SUCCESS; CloseRegionProcedure 15f8962506199e1be1a11e7189a213ff, server=fad905551524,38809,1733834007617 in 1.8490 sec 2024-12-10T12:35:30,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=122, resume processing ppid=121 2024-12-10T12:35:30,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=122, ppid=121, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=15f8962506199e1be1a11e7189a213ff, UNASSIGN in 1.8540 sec 2024-12-10T12:35:30,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=121, resume processing ppid=120 2024-12-10T12:35:30,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=121, ppid=120, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8580 sec 2024-12-10T12:35:30,325 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834130325"}]},"ts":"1733834130325"} 2024-12-10T12:35:30,326 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T12:35:30,372 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T12:35:30,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=120, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.9580 sec 2024-12-10T12:35:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-12-10T12:35:30,527 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 120 completed 2024-12-10T12:35:30,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T12:35:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=124, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:30,532 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=124, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-12-10T12:35:30,534 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=124, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:30,536 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,542 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/recovered.edits] 2024-12-10T12:35:30,548 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/6d0eaff45d504243bfeaa66573650c43 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/6d0eaff45d504243bfeaa66573650c43 2024-12-10T12:35:30,548 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c4940c3752e040f3b3169de34ee0714e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c4940c3752e040f3b3169de34ee0714e 2024-12-10T12:35:30,548 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db65bb2f736c4925aeb9ea72ed5a0824 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/db65bb2f736c4925aeb9ea72ed5a0824 2024-12-10T12:35:30,549 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c305ddf3a6394f1abd2dd8fa63a7792e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/A/c305ddf3a6394f1abd2dd8fa63a7792e 2024-12-10T12:35:30,554 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/10e3705c24cd44d4b5990c22badfb644 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/10e3705c24cd44d4b5990c22badfb644 2024-12-10T12:35:30,554 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/532eca5fd3634fe084caa8db24a5d647 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/532eca5fd3634fe084caa8db24a5d647 2024-12-10T12:35:30,554 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/88123981144445a0bd917555918ec0c6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/88123981144445a0bd917555918ec0c6 2024-12-10T12:35:30,554 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/e76ec2f680d44376ad039829b88fc76d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/B/e76ec2f680d44376ad039829b88fc76d 2024-12-10T12:35:30,558 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/07f018412f3144ada2065f147ca7ad4b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/07f018412f3144ada2065f147ca7ad4b 2024-12-10T12:35:30,558 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c7c7999a98be43e496757c8e486c7cfa to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c7c7999a98be43e496757c8e486c7cfa 2024-12-10T12:35:30,558 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/402d118ab6f04ebbbce541081ecfe740 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/402d118ab6f04ebbbce541081ecfe740 2024-12-10T12:35:30,558 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c2cab22af82f41bfbd87e6180c14cf1e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/C/c2cab22af82f41bfbd87e6180c14cf1e 2024-12-10T12:35:30,561 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/recovered.edits/289.seqid to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff/recovered.edits/289.seqid 2024-12-10T12:35:30,562 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,562 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T12:35:30,562 DEBUG [PEWorker-1 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T12:35:30,564 DEBUG [PEWorker-1 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T12:35:30,572 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210033ef0dda15346609f8507345b7b9b36_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210033ef0dda15346609f8507345b7b9b36_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,572 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100bae5366d2df46f9a8d4026f58078553_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100bae5366d2df46f9a8d4026f58078553_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,572 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101cb52d42a0c442c7a1675f3608eefd46_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101cb52d42a0c442c7a1675f3608eefd46_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,573 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102ecbe7126d274c3faa2119331d82cc53_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102ecbe7126d274c3faa2119331d82cc53_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,573 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210306a0077ee0e44778b49f9fbc2cdfd94_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210306a0077ee0e44778b49f9fbc2cdfd94_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,573 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121050d532c6b0ab440d852cbf48bead819f_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121050d532c6b0ab440d852cbf48bead819f_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,573 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121053e13a0240bb49dab651506f52f2e95c_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121053e13a0240bb49dab651506f52f2e95c_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,573 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107d59a3c024384e60815265d8767e24ba_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107d59a3c024384e60815265d8767e24ba_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,574 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108f8b26632d7a40db95b52cf626541b56_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108f8b26632d7a40db95b52cf626541b56_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,574 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109eaa590a71124320b06f105ee3094d63_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109eaa590a71124320b06f105ee3094d63_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,575 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109dc9d7162148498481f99475fff416c8_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109dc9d7162148498481f99475fff416c8_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,575 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d048e219340a42aabdf1bd67989387ae_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d048e219340a42aabdf1bd67989387ae_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,575 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cc50331479bd4ff791a63a6ab39538f2_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cc50331479bd4ff791a63a6ab39538f2_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,575 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109d84036311c542c3a479c39a90f70fb5_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109d84036311c542c3a479c39a90f70fb5_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,575 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f6a50513e5ae4d37b265e6c27a6725df_15f8962506199e1be1a11e7189a213ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f6a50513e5ae4d37b265e6c27a6725df_15f8962506199e1be1a11e7189a213ff 2024-12-10T12:35:30,576 DEBUG [PEWorker-1 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T12:35:30,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=124, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:30,580 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T12:35:30,582 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T12:35:30,583 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=124, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:30,583 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T12:35:30,584 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733834130583"}]},"ts":"9223372036854775807"} 2024-12-10T12:35:30,585 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-12-10T12:35:30,586 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 15f8962506199e1be1a11e7189a213ff, NAME => 'TestAcidGuarantees,,1733834103405.15f8962506199e1be1a11e7189a213ff.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T12:35:30,586 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T12:35:30,586 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733834130586"}]},"ts":"9223372036854775807"} 2024-12-10T12:35:30,587 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-12-10T12:35:30,597 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=124, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:30,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=124, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-12-10T12:35:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-12-10T12:35:30,635 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 124 completed 2024-12-10T12:35:30,650 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMobScanAtomicity Thread=245 (was 245), OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=295 (was 261) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9432 (was 9448) 2024-12-10T12:35:30,659 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testScanAtomicity Thread=245, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=295, ProcessCount=11, AvailableMemoryMB=9432 2024-12-10T12:35:30,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T12:35:30,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:35:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=125, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:30,662 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=125, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T12:35:30,662 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:30,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 125 2024-12-10T12:35:30,663 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=125, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T12:35:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-10T12:35:30,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742228_1404 (size=963) 2024-12-10T12:35:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-10T12:35:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-10T12:35:31,074 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 2024-12-10T12:35:31,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742229_1405 (size=53) 2024-12-10T12:35:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-10T12:35:31,484 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:35:31,484 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing 19609dde5fb8a6633d70503eaf3922b6, disabling compactions & flushes 2024-12-10T12:35:31,484 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:31,484 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:31,484 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. after waiting 0 ms 2024-12-10T12:35:31,484 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:31,484 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:31,484 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:31,485 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=125, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T12:35:31,486 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733834131485"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733834131485"}]},"ts":"1733834131485"} 2024-12-10T12:35:31,487 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-10T12:35:31,488 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=125, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T12:35:31,488 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834131488"}]},"ts":"1733834131488"} 2024-12-10T12:35:31,489 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T12:35:31,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=19609dde5fb8a6633d70503eaf3922b6, ASSIGN}] 2024-12-10T12:35:31,578 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=126, ppid=125, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=19609dde5fb8a6633d70503eaf3922b6, ASSIGN 2024-12-10T12:35:31,579 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=126, ppid=125, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=19609dde5fb8a6633d70503eaf3922b6, ASSIGN; state=OFFLINE, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=false 2024-12-10T12:35:31,730 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=126 updating hbase:meta row=19609dde5fb8a6633d70503eaf3922b6, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:31,732 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE; OpenRegionProcedure 19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-10T12:35:31,884 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:31,920 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:31,921 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(7285): Opening region: {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:35:31,921 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:31,922 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:35:31,922 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(7327): checking encryption for 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:31,922 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(7330): checking classloading for 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:31,924 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:31,927 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:31,927 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19609dde5fb8a6633d70503eaf3922b6 columnFamilyName A 2024-12-10T12:35:31,927 DEBUG [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:31,928 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] regionserver.HStore(327): Store=19609dde5fb8a6633d70503eaf3922b6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:31,929 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:31,931 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:31,931 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19609dde5fb8a6633d70503eaf3922b6 columnFamilyName B 2024-12-10T12:35:31,931 DEBUG [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:31,932 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] regionserver.HStore(327): Store=19609dde5fb8a6633d70503eaf3922b6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:31,932 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:31,934 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:31,934 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19609dde5fb8a6633d70503eaf3922b6 columnFamilyName C 2024-12-10T12:35:31,934 DEBUG [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:31,934 INFO [StoreOpener-19609dde5fb8a6633d70503eaf3922b6-1 {}] regionserver.HStore(327): Store=19609dde5fb8a6633d70503eaf3922b6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:31,934 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:31,935 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:31,935 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:31,937 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:35:31,938 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(1085): writing seq id for 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:31,940 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:35:31,940 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(1102): Opened 19609dde5fb8a6633d70503eaf3922b6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75164654, jitterRate=0.12004062533378601}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:35:31,941 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegion(1001): Region open journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:31,942 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., pid=127, masterSystemTime=1733834131884 2024-12-10T12:35:31,943 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:31,943 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=127}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:31,943 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=126 updating hbase:meta row=19609dde5fb8a6633d70503eaf3922b6, regionState=OPEN, openSeqNum=2, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:31,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=127, resume processing ppid=126 2024-12-10T12:35:31,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=127, ppid=126, state=SUCCESS; OpenRegionProcedure 19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 in 212 msec 2024-12-10T12:35:31,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=126, resume processing ppid=125 2024-12-10T12:35:31,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=126, ppid=125, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=19609dde5fb8a6633d70503eaf3922b6, ASSIGN in 371 msec 2024-12-10T12:35:31,948 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=125, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T12:35:31,948 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834131948"}]},"ts":"1733834131948"} 2024-12-10T12:35:31,949 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T12:35:31,956 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=125, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T12:35:31,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=125, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2950 sec 2024-12-10T12:35:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-10T12:35:32,773 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 125 completed 2024-12-10T12:35:32,776 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7177efc9 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@65df2359 2024-12-10T12:35:32,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef40578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,831 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39570, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T12:35:32,835 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54020, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T12:35:32,837 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x61d38088 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d0ab200 2024-12-10T12:35:32,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32bb71c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,848 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7043f683 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5871c039 2024-12-10T12:35:32,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc0f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,857 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b0c2472 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7daa5922 2024-12-10T12:35:32,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8b6e04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,866 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34b30c39 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b7f20c4 2024-12-10T12:35:32,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc486e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,875 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d672ed2 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f7c40ba 2024-12-10T12:35:32,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2070263a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,881 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x41b0e7b6 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6050584c 2024-12-10T12:35:32,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154f0f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,889 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f2423f3 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6dd48863 2024-12-10T12:35:32,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a917b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,898 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x184771cf to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51196534 2024-12-10T12:35:32,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c2725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,906 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x076f0408 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1dc5e114 2024-12-10T12:35:32,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d49886, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,915 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c692575 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e96b8ad 2024-12-10T12:35:32,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635b1751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:32,924 DEBUG [hconnection-0x62d64946-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,924 DEBUG [hconnection-0x538a3640-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,925 DEBUG [hconnection-0x7564581d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,925 DEBUG [hconnection-0x7dfc6b37-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:32,925 DEBUG [hconnection-0x369c5f15-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,926 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39588, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,926 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39578, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,926 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39632, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,926 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39618, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,927 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39602, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=128, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees 2024-12-10T12:35:32,928 DEBUG [hconnection-0x35c5addb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,928 DEBUG [hconnection-0x1ed2dc26-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,928 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=128, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:32,928 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39636, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,929 DEBUG [hconnection-0x4f11267e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,929 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39638, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,929 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=128, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-12-10T12:35:32,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:32,930 DEBUG [hconnection-0x692f6ff2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,930 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39646, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,930 DEBUG [hconnection-0x4fe69aef-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:32,931 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39648, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,931 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39664, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:32,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:35:32,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:32,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:32,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:32,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:32,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:32,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:32,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834192943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:32,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834192944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:32,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834192944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:32,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834192945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:32,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834192949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:32,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/9189aa55fe9f4c2082ee7771ec4afbe7 is 50, key is test_row_0/A:col10/1733834132931/Put/seqid=0 2024-12-10T12:35:32,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742230_1406 (size=12001) 2024-12-10T12:35:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-12-10T12:35:33,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834193045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834193048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834193048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834193050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834193052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,083 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:33,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-12-10T12:35:33,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:33,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-12-10T12:35:33,235 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:33,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-12-10T12:35:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834193247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834193250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834193250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834193251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834193255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/9189aa55fe9f4c2082ee7771ec4afbe7 2024-12-10T12:35:33,388 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:33,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-12-10T12:35:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/1a3b1398f17f4661acf598f426efb8ab is 50, key is test_row_0/B:col10/1733834132931/Put/seqid=0 2024-12-10T12:35:33,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742231_1407 (size=12001) 2024-12-10T12:35:33,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/1a3b1398f17f4661acf598f426efb8ab 2024-12-10T12:35:33,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/7cb21b6ab4524eb7b12c443369657308 is 50, key is test_row_0/C:col10/1733834132931/Put/seqid=0 2024-12-10T12:35:33,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742232_1408 (size=12001) 2024-12-10T12:35:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-12-10T12:35:33,540 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:33,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-12-10T12:35:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834193552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834193553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834193553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834193556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834193559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:33,692 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:33,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-12-10T12:35:33,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:33,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:33,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/7cb21b6ab4524eb7b12c443369657308 2024-12-10T12:35:33,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/9189aa55fe9f4c2082ee7771ec4afbe7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9189aa55fe9f4c2082ee7771ec4afbe7 2024-12-10T12:35:33,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9189aa55fe9f4c2082ee7771ec4afbe7, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T12:35:33,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/1a3b1398f17f4661acf598f426efb8ab as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/1a3b1398f17f4661acf598f426efb8ab 2024-12-10T12:35:33,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/1a3b1398f17f4661acf598f426efb8ab, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T12:35:33,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/7cb21b6ab4524eb7b12c443369657308 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/7cb21b6ab4524eb7b12c443369657308 2024-12-10T12:35:33,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/7cb21b6ab4524eb7b12c443369657308, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T12:35:33,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 19609dde5fb8a6633d70503eaf3922b6 in 896ms, sequenceid=14, compaction requested=false 2024-12-10T12:35:33,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:33,844 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:33,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-12-10T12:35:33,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:33,845 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-10T12:35:33,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:33,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:33,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:33,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:33,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:33,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/d751b87d865e4928b31cf8c6449543e5 is 50, key is test_row_0/A:col10/1733834132943/Put/seqid=0 2024-12-10T12:35:33,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742233_1409 (size=12001) 2024-12-10T12:35:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-12-10T12:35:34,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:34,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834194064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834194065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834194065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834194066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834194066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834194169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834194170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834194170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834194170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834194171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,256 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/d751b87d865e4928b31cf8c6449543e5 2024-12-10T12:35:34,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/9c488a92b1c54ef9bd2f7c8af30e27c3 is 50, key is test_row_0/B:col10/1733834132943/Put/seqid=0 2024-12-10T12:35:34,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742234_1410 (size=12001) 2024-12-10T12:35:34,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834194373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834194373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834194373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834194374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834194374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,666 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/9c488a92b1c54ef9bd2f7c8af30e27c3 2024-12-10T12:35:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/b3f9123045b34ae7b0c7708fc58bce59 is 50, key is test_row_0/C:col10/1733834132943/Put/seqid=0 2024-12-10T12:35:34,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742235_1411 (size=12001) 2024-12-10T12:35:34,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834194676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834194677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834194677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834194678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:34,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834194679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-12-10T12:35:35,076 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/b3f9123045b34ae7b0c7708fc58bce59 2024-12-10T12:35:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/d751b87d865e4928b31cf8c6449543e5 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/d751b87d865e4928b31cf8c6449543e5 2024-12-10T12:35:35,082 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/d751b87d865e4928b31cf8c6449543e5, entries=150, sequenceid=38, filesize=11.7 K 2024-12-10T12:35:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/9c488a92b1c54ef9bd2f7c8af30e27c3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/9c488a92b1c54ef9bd2f7c8af30e27c3 2024-12-10T12:35:35,086 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/9c488a92b1c54ef9bd2f7c8af30e27c3, entries=150, sequenceid=38, filesize=11.7 K 2024-12-10T12:35:35,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/b3f9123045b34ae7b0c7708fc58bce59 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3f9123045b34ae7b0c7708fc58bce59 2024-12-10T12:35:35,089 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3f9123045b34ae7b0c7708fc58bce59, entries=150, sequenceid=38, filesize=11.7 K 2024-12-10T12:35:35,090 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 19609dde5fb8a6633d70503eaf3922b6 in 1246ms, sequenceid=38, compaction requested=false 2024-12-10T12:35:35,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:35,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:35,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129 2024-12-10T12:35:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=129 2024-12-10T12:35:35,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=129, resume processing ppid=128 2024-12-10T12:35:35,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=129, ppid=128, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1620 sec 2024-12-10T12:35:35,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=128, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees in 2.1670 sec 2024-12-10T12:35:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:35,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:35:35,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:35,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:35,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:35,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:35,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:35,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:35,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/cdaba4e9f64b4f3f8dbb03aa2634b424 is 50, key is test_row_0/A:col10/1733834135181/Put/seqid=0 2024-12-10T12:35:35,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742236_1412 (size=16681) 2024-12-10T12:35:35,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834195198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834195201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834195203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834195204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834195204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834195304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834195305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834195308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834195308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834195309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834195509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834195509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834195512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834195512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834195513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/cdaba4e9f64b4f3f8dbb03aa2634b424 2024-12-10T12:35:35,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/aad3d48420d542ce98a4ca17bf5f261f is 50, key is test_row_0/B:col10/1733834135181/Put/seqid=0 2024-12-10T12:35:35,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742237_1413 (size=12001) 2024-12-10T12:35:35,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834195812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834195814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834195816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834195816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834195819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:35,841 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T12:35:36,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/aad3d48420d542ce98a4ca17bf5f261f 2024-12-10T12:35:36,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/90666f58541548f492bb4cc2b983baa3 is 50, key is test_row_0/C:col10/1733834135181/Put/seqid=0 2024-12-10T12:35:36,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742238_1414 (size=12001) 2024-12-10T12:35:36,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834196318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:36,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834196318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:36,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834196322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:36,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834196323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:36,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834196323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:36,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/90666f58541548f492bb4cc2b983baa3 2024-12-10T12:35:36,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/cdaba4e9f64b4f3f8dbb03aa2634b424 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/cdaba4e9f64b4f3f8dbb03aa2634b424 2024-12-10T12:35:36,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/cdaba4e9f64b4f3f8dbb03aa2634b424, entries=250, sequenceid=52, filesize=16.3 K 2024-12-10T12:35:36,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/aad3d48420d542ce98a4ca17bf5f261f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aad3d48420d542ce98a4ca17bf5f261f 2024-12-10T12:35:36,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aad3d48420d542ce98a4ca17bf5f261f, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T12:35:36,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/90666f58541548f492bb4cc2b983baa3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/90666f58541548f492bb4cc2b983baa3 2024-12-10T12:35:36,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/90666f58541548f492bb4cc2b983baa3, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T12:35:36,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 19609dde5fb8a6633d70503eaf3922b6 in 1243ms, sequenceid=52, compaction requested=true 2024-12-10T12:35:36,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:36,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:36,424 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:36,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:36,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:36,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:36,424 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:36,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:36,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:36,425 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:36,425 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:36,425 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/B is initiating minor compaction (all files) 2024-12-10T12:35:36,425 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/A is initiating minor compaction (all files) 2024-12-10T12:35:36,425 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/A in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:36,425 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/B in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:36,426 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9189aa55fe9f4c2082ee7771ec4afbe7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/d751b87d865e4928b31cf8c6449543e5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/cdaba4e9f64b4f3f8dbb03aa2634b424] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=39.7 K 2024-12-10T12:35:36,426 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/1a3b1398f17f4661acf598f426efb8ab, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/9c488a92b1c54ef9bd2f7c8af30e27c3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aad3d48420d542ce98a4ca17bf5f261f] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=35.2 K 2024-12-10T12:35:36,426 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9189aa55fe9f4c2082ee7771ec4afbe7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733834132931 2024-12-10T12:35:36,426 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1a3b1398f17f4661acf598f426efb8ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733834132931 2024-12-10T12:35:36,426 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting d751b87d865e4928b31cf8c6449543e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733834132943 2024-12-10T12:35:36,426 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 9c488a92b1c54ef9bd2f7c8af30e27c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733834132943 2024-12-10T12:35:36,426 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting cdaba4e9f64b4f3f8dbb03aa2634b424, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834134063 2024-12-10T12:35:36,426 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting aad3d48420d542ce98a4ca17bf5f261f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834134063 2024-12-10T12:35:36,431 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#B#compaction#345 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:36,432 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/4a37e41d17da4934af08bc6b1fd1e716 is 50, key is test_row_0/B:col10/1733834135181/Put/seqid=0 2024-12-10T12:35:36,432 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#A#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:36,432 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/c249720c102645128f93f519938fe744 is 50, key is test_row_0/A:col10/1733834135181/Put/seqid=0 2024-12-10T12:35:36,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742239_1415 (size=12104) 2024-12-10T12:35:36,440 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/4a37e41d17da4934af08bc6b1fd1e716 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/4a37e41d17da4934af08bc6b1fd1e716 2024-12-10T12:35:36,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742240_1416 (size=12104) 2024-12-10T12:35:36,447 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/B of 19609dde5fb8a6633d70503eaf3922b6 into 4a37e41d17da4934af08bc6b1fd1e716(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:36,447 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:36,447 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/B, priority=13, startTime=1733834136424; duration=0sec 2024-12-10T12:35:36,447 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:36,447 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:B 2024-12-10T12:35:36,447 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:36,448 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:36,448 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/C is initiating minor compaction (all files) 2024-12-10T12:35:36,448 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/C in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:36,448 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/7cb21b6ab4524eb7b12c443369657308, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3f9123045b34ae7b0c7708fc58bce59, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/90666f58541548f492bb4cc2b983baa3] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=35.2 K 2024-12-10T12:35:36,449 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 7cb21b6ab4524eb7b12c443369657308, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733834132931 2024-12-10T12:35:36,449 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting b3f9123045b34ae7b0c7708fc58bce59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733834132943 2024-12-10T12:35:36,449 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 90666f58541548f492bb4cc2b983baa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834134063 2024-12-10T12:35:36,453 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/c249720c102645128f93f519938fe744 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c249720c102645128f93f519938fe744 2024-12-10T12:35:36,457 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/A of 19609dde5fb8a6633d70503eaf3922b6 into c249720c102645128f93f519938fe744(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:36,457 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:36,457 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/A, priority=13, startTime=1733834136424; duration=0sec 2024-12-10T12:35:36,457 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:36,458 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:A 2024-12-10T12:35:36,458 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:36,458 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/5ae05380d175463c990feac12035b7df is 50, key is test_row_0/C:col10/1733834135181/Put/seqid=0 2024-12-10T12:35:36,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742241_1417 (size=12104) 2024-12-10T12:35:36,866 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/5ae05380d175463c990feac12035b7df as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5ae05380d175463c990feac12035b7df 2024-12-10T12:35:36,869 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/C of 19609dde5fb8a6633d70503eaf3922b6 into 5ae05380d175463c990feac12035b7df(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:36,869 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:36,870 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/C, priority=13, startTime=1733834136424; duration=0sec 2024-12-10T12:35:36,870 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:36,870 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:C 2024-12-10T12:35:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-12-10T12:35:37,033 INFO [Thread-1832 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 128 completed 2024-12-10T12:35:37,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees 2024-12-10T12:35:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-10T12:35:37,035 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:37,036 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:37,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-10T12:35:37,187 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:37,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-12-10T12:35:37,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:37,187 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-10T12:35:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/c105a2c296124a1a9f21650a88d4cc14 is 50, key is test_row_0/A:col10/1733834135189/Put/seqid=0 2024-12-10T12:35:37,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742242_1418 (size=12001) 2024-12-10T12:35:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:37,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-10T12:35:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834197337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834197338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834197338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834197339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834197339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834197440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834197443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834197443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834197443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834197443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,595 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/c105a2c296124a1a9f21650a88d4cc14 2024-12-10T12:35:37,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/f8d6dc06114346bfb911c806b15101d2 is 50, key is test_row_0/B:col10/1733834135189/Put/seqid=0 2024-12-10T12:35:37,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742243_1419 (size=12001) 2024-12-10T12:35:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-10T12:35:37,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834197643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834197644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834197644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834197644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834197647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834197946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834197947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834197948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834197948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834197951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,004 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/f8d6dc06114346bfb911c806b15101d2 2024-12-10T12:35:38,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/1233887f37cb4fb598e74ab4c28a2974 is 50, key is test_row_0/C:col10/1733834135189/Put/seqid=0 2024-12-10T12:35:38,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742244_1420 (size=12001) 2024-12-10T12:35:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-10T12:35:38,413 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/1233887f37cb4fb598e74ab4c28a2974 2024-12-10T12:35:38,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/c105a2c296124a1a9f21650a88d4cc14 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c105a2c296124a1a9f21650a88d4cc14 2024-12-10T12:35:38,419 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c105a2c296124a1a9f21650a88d4cc14, entries=150, sequenceid=78, filesize=11.7 K 2024-12-10T12:35:38,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/f8d6dc06114346bfb911c806b15101d2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/f8d6dc06114346bfb911c806b15101d2 2024-12-10T12:35:38,422 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/f8d6dc06114346bfb911c806b15101d2, entries=150, sequenceid=78, filesize=11.7 K 2024-12-10T12:35:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/1233887f37cb4fb598e74ab4c28a2974 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/1233887f37cb4fb598e74ab4c28a2974 2024-12-10T12:35:38,426 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/1233887f37cb4fb598e74ab4c28a2974, entries=150, sequenceid=78, filesize=11.7 K 2024-12-10T12:35:38,427 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 19609dde5fb8a6633d70503eaf3922b6 in 1240ms, sequenceid=78, compaction requested=false 2024-12-10T12:35:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-12-10T12:35:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=131 2024-12-10T12:35:38,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=131, resume processing ppid=130 2024-12-10T12:35:38,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=131, ppid=130, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3920 sec 2024-12-10T12:35:38,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=130, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees in 1.3950 sec 2024-12-10T12:35:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:38,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-10T12:35:38,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:38,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:38,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:38,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:38,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:38,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:38,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/0e0982a791764b3197ff10273b36d517 is 50, key is test_row_0/A:col10/1733834137337/Put/seqid=0 2024-12-10T12:35:38,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742245_1421 (size=14341) 2024-12-10T12:35:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834198471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834198472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834198474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834198474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834198475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834198577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834198577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834198578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834198580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834198580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834198780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834198781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834198782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834198783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834198785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:38,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/0e0982a791764b3197ff10273b36d517 2024-12-10T12:35:38,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/77c3e1ffb1cc4bcd98fe51d40f9eb6e4 is 50, key is test_row_0/B:col10/1733834137337/Put/seqid=0 2024-12-10T12:35:38,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742246_1422 (size=12001) 2024-12-10T12:35:39,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834199082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834199087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834199087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834199087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834199090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-10T12:35:39,138 INFO [Thread-1832 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 130 completed 2024-12-10T12:35:39,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2024-12-10T12:35:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-10T12:35:39,140 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:39,141 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:39,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-10T12:35:39,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/77c3e1ffb1cc4bcd98fe51d40f9eb6e4 2024-12-10T12:35:39,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/d838b18f472046058752a52409b5370c is 50, key is test_row_0/C:col10/1733834137337/Put/seqid=0 2024-12-10T12:35:39,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742247_1423 (size=12001) 2024-12-10T12:35:39,292 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:39,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T12:35:39,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:39,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:39,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:39,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:39,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-10T12:35:39,444 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:39,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T12:35:39,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:39,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:39,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:39,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:39,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834199586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834199589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834199590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834199592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834199592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:39,596 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:39,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T12:35:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:39,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:39,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/d838b18f472046058752a52409b5370c 2024-12-10T12:35:39,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/0e0982a791764b3197ff10273b36d517 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0e0982a791764b3197ff10273b36d517 2024-12-10T12:35:39,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0e0982a791764b3197ff10273b36d517, entries=200, sequenceid=93, filesize=14.0 K 2024-12-10T12:35:39,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/77c3e1ffb1cc4bcd98fe51d40f9eb6e4 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/77c3e1ffb1cc4bcd98fe51d40f9eb6e4 2024-12-10T12:35:39,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/77c3e1ffb1cc4bcd98fe51d40f9eb6e4, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T12:35:39,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/d838b18f472046058752a52409b5370c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/d838b18f472046058752a52409b5370c 2024-12-10T12:35:39,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/d838b18f472046058752a52409b5370c, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T12:35:39,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 19609dde5fb8a6633d70503eaf3922b6 in 1250ms, sequenceid=93, compaction requested=true 2024-12-10T12:35:39,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:39,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:39,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:39,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:39,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:39,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:39,706 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:39,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:39,706 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:39,706 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:39,706 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:39,706 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/A is initiating minor compaction (all files) 2024-12-10T12:35:39,706 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/B is initiating minor compaction (all files) 2024-12-10T12:35:39,706 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/A in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:39,706 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/B in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:39,706 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/4a37e41d17da4934af08bc6b1fd1e716, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/f8d6dc06114346bfb911c806b15101d2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/77c3e1ffb1cc4bcd98fe51d40f9eb6e4] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=35.3 K 2024-12-10T12:35:39,706 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c249720c102645128f93f519938fe744, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c105a2c296124a1a9f21650a88d4cc14, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0e0982a791764b3197ff10273b36d517] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=37.5 K 2024-12-10T12:35:39,707 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 4a37e41d17da4934af08bc6b1fd1e716, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834134063 2024-12-10T12:35:39,707 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting c249720c102645128f93f519938fe744, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834134063 2024-12-10T12:35:39,707 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting c105a2c296124a1a9f21650a88d4cc14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733834135189 2024-12-10T12:35:39,707 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting f8d6dc06114346bfb911c806b15101d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733834135189 2024-12-10T12:35:39,707 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0e0982a791764b3197ff10273b36d517, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733834137337 2024-12-10T12:35:39,707 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 77c3e1ffb1cc4bcd98fe51d40f9eb6e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733834137337 2024-12-10T12:35:39,712 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:39,713 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/dda3e8b2201147c4a265779c97e3d669 is 50, key is test_row_0/B:col10/1733834137337/Put/seqid=0 2024-12-10T12:35:39,715 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#A#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:39,716 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/dfc2e12a9082496085f9e585ab4b8efa is 50, key is test_row_0/A:col10/1733834137337/Put/seqid=0 2024-12-10T12:35:39,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742248_1424 (size=12207) 2024-12-10T12:35:39,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742249_1425 (size=12207) 2024-12-10T12:35:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-10T12:35:39,748 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:39,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T12:35:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:39,749 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-10T12:35:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:39,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/2db61739088745f5ab02f7d3b8804665 is 50, key is test_row_0/A:col10/1733834138470/Put/seqid=0 2024-12-10T12:35:39,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742250_1426 (size=12001) 2024-12-10T12:35:40,129 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/dda3e8b2201147c4a265779c97e3d669 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dda3e8b2201147c4a265779c97e3d669 2024-12-10T12:35:40,129 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/dfc2e12a9082496085f9e585ab4b8efa as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/dfc2e12a9082496085f9e585ab4b8efa 2024-12-10T12:35:40,132 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/A of 19609dde5fb8a6633d70503eaf3922b6 into dfc2e12a9082496085f9e585ab4b8efa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:40,133 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/B of 19609dde5fb8a6633d70503eaf3922b6 into dda3e8b2201147c4a265779c97e3d669(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:40,133 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:40,133 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:40,133 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/A, priority=13, startTime=1733834139705; duration=0sec 2024-12-10T12:35:40,133 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/B, priority=13, startTime=1733834139706; duration=0sec 2024-12-10T12:35:40,133 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:40,133 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:A 2024-12-10T12:35:40,133 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:40,133 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:B 2024-12-10T12:35:40,133 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:40,133 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:40,133 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/C is initiating minor compaction (all files) 2024-12-10T12:35:40,134 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/C in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:40,134 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5ae05380d175463c990feac12035b7df, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/1233887f37cb4fb598e74ab4c28a2974, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/d838b18f472046058752a52409b5370c] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=35.3 K 2024-12-10T12:35:40,134 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5ae05380d175463c990feac12035b7df, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733834134063 2024-12-10T12:35:40,134 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1233887f37cb4fb598e74ab4c28a2974, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733834135189 2024-12-10T12:35:40,134 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting d838b18f472046058752a52409b5370c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733834137337 2024-12-10T12:35:40,138 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#C#compaction#357 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:40,139 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/2a43868c6a1840308c4bb8dd67309ba1 is 50, key is test_row_0/C:col10/1733834137337/Put/seqid=0 2024-12-10T12:35:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742251_1427 (size=12207) 2024-12-10T12:35:40,152 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/2a43868c6a1840308c4bb8dd67309ba1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/2a43868c6a1840308c4bb8dd67309ba1 2024-12-10T12:35:40,155 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/C of 19609dde5fb8a6633d70503eaf3922b6 into 2a43868c6a1840308c4bb8dd67309ba1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:40,155 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:40,155 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/C, priority=13, startTime=1733834139706; duration=0sec 2024-12-10T12:35:40,155 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:40,155 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:C 2024-12-10T12:35:40,156 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/2db61739088745f5ab02f7d3b8804665 2024-12-10T12:35:40,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/7186964619264217bdc1eb08602edb90 is 50, key is test_row_0/B:col10/1733834138470/Put/seqid=0 2024-12-10T12:35:40,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742252_1428 (size=12001) 2024-12-10T12:35:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-10T12:35:40,569 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/7186964619264217bdc1eb08602edb90 2024-12-10T12:35:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/a2d854b2c02c49e780f2ad0496291a5b is 50, key is test_row_0/C:col10/1733834138470/Put/seqid=0 2024-12-10T12:35:40,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742253_1429 (size=12001) 2024-12-10T12:35:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:40,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834200602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834200605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834200606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834200606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834200606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834200708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834200709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834200709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834200711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834200712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834200911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834200912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834200912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834200914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834200915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:40,984 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/a2d854b2c02c49e780f2ad0496291a5b 2024-12-10T12:35:40,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/2db61739088745f5ab02f7d3b8804665 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/2db61739088745f5ab02f7d3b8804665 2024-12-10T12:35:40,990 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/2db61739088745f5ab02f7d3b8804665, entries=150, sequenceid=114, filesize=11.7 K 2024-12-10T12:35:40,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/7186964619264217bdc1eb08602edb90 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/7186964619264217bdc1eb08602edb90 2024-12-10T12:35:40,994 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/7186964619264217bdc1eb08602edb90, entries=150, sequenceid=114, filesize=11.7 K 2024-12-10T12:35:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/a2d854b2c02c49e780f2ad0496291a5b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/a2d854b2c02c49e780f2ad0496291a5b 2024-12-10T12:35:40,997 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/a2d854b2c02c49e780f2ad0496291a5b, entries=150, sequenceid=114, filesize=11.7 K 2024-12-10T12:35:40,998 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 19609dde5fb8a6633d70503eaf3922b6 in 1249ms, sequenceid=114, compaction requested=false 2024-12-10T12:35:40,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:40,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:40,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=133 2024-12-10T12:35:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=133 2024-12-10T12:35:41,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=133, resume processing ppid=132 2024-12-10T12:35:41,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=133, ppid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8580 sec 2024-12-10T12:35:41,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 1.8610 sec 2024-12-10T12:35:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:41,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-10T12:35:41,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:41,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:41,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:41,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:41,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:41,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:41,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/21b8afca1c96497e8117b5ec7c592ea1 is 50, key is test_row_0/A:col10/1733834141214/Put/seqid=0 2024-12-10T12:35:41,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742254_1430 (size=14491) 2024-12-10T12:35:41,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834201234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834201234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834201235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834201239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-10T12:35:41,243 INFO [Thread-1832 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 132 completed 2024-12-10T12:35:41,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834201239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2024-12-10T12:35:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-12-10T12:35:41,245 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:41,246 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:41,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:41,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834201340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834201340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834201340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-12-10T12:35:41,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834201343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834201344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,397 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:41,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T12:35:41,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:41,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:41,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:41,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834201543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-12-10T12:35:41,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834201544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834201544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,549 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:41,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T12:35:41,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:41,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:41,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:41,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834201548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834201549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/21b8afca1c96497e8117b5ec7c592ea1 2024-12-10T12:35:41,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/55ee374278dc4cb6b04c79bc99729bf6 is 50, key is test_row_0/B:col10/1733834141214/Put/seqid=0 2024-12-10T12:35:41,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742255_1431 (size=12151) 2024-12-10T12:35:41,701 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:41,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T12:35:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:41,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-12-10T12:35:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834201848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834201848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834201849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,853 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:41,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T12:35:41,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:41,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:41,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:41,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:41,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834201855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:41,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834201855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:42,005 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:42,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T12:35:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/55ee374278dc4cb6b04c79bc99729bf6 2024-12-10T12:35:42,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/3812502b39f84d7887ba015f2e425dab is 50, key is test_row_0/C:col10/1733834141214/Put/seqid=0 2024-12-10T12:35:42,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742256_1432 (size=12151) 2024-12-10T12:35:42,158 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:42,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T12:35:42,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:42,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,310 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:42,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T12:35:42,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-12-10T12:35:42,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834202352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:42,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834202353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:42,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834202353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:42,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834202360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:42,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834202360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:42,462 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:42,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T12:35:42,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:42,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:42,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/3812502b39f84d7887ba015f2e425dab 2024-12-10T12:35:42,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/21b8afca1c96497e8117b5ec7c592ea1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/21b8afca1c96497e8117b5ec7c592ea1 2024-12-10T12:35:42,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/21b8afca1c96497e8117b5ec7c592ea1, entries=200, sequenceid=133, filesize=14.2 K 2024-12-10T12:35:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/55ee374278dc4cb6b04c79bc99729bf6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/55ee374278dc4cb6b04c79bc99729bf6 2024-12-10T12:35:42,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/55ee374278dc4cb6b04c79bc99729bf6, entries=150, sequenceid=133, filesize=11.9 K 2024-12-10T12:35:42,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/3812502b39f84d7887ba015f2e425dab as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/3812502b39f84d7887ba015f2e425dab 2024-12-10T12:35:42,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/3812502b39f84d7887ba015f2e425dab, entries=150, sequenceid=133, filesize=11.9 K 2024-12-10T12:35:42,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 19609dde5fb8a6633d70503eaf3922b6 in 1287ms, sequenceid=133, compaction requested=true 2024-12-10T12:35:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:42,503 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:42,503 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:42,503 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:42,504 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/B is initiating minor compaction (all files) 2024-12-10T12:35:42,504 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/B in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,504 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:42,504 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/A is initiating minor compaction (all files) 2024-12-10T12:35:42,504 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dda3e8b2201147c4a265779c97e3d669, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/7186964619264217bdc1eb08602edb90, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/55ee374278dc4cb6b04c79bc99729bf6] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=35.5 K 2024-12-10T12:35:42,504 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/A in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,504 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/dfc2e12a9082496085f9e585ab4b8efa, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/2db61739088745f5ab02f7d3b8804665, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/21b8afca1c96497e8117b5ec7c592ea1] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=37.8 K 2024-12-10T12:35:42,504 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting dda3e8b2201147c4a265779c97e3d669, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733834137337 2024-12-10T12:35:42,504 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting dfc2e12a9082496085f9e585ab4b8efa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733834137337 2024-12-10T12:35:42,504 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 7186964619264217bdc1eb08602edb90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733834138470 2024-12-10T12:35:42,504 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2db61739088745f5ab02f7d3b8804665, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733834138470 2024-12-10T12:35:42,504 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 55ee374278dc4cb6b04c79bc99729bf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733834140604 2024-12-10T12:35:42,504 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 21b8afca1c96497e8117b5ec7c592ea1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733834140601 2024-12-10T12:35:42,510 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:42,510 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/ca5e4126f8404ea69f79ed0bab6fe34b is 50, key is test_row_0/B:col10/1733834141214/Put/seqid=0 2024-12-10T12:35:42,512 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#A#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:42,512 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/4d19b967d3994aac8a7c0747531c96ec is 50, key is test_row_0/A:col10/1733834141214/Put/seqid=0 2024-12-10T12:35:42,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742257_1433 (size=12459) 2024-12-10T12:35:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742258_1434 (size=12459) 2024-12-10T12:35:42,518 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/4d19b967d3994aac8a7c0747531c96ec as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/4d19b967d3994aac8a7c0747531c96ec 2024-12-10T12:35:42,521 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/A of 19609dde5fb8a6633d70503eaf3922b6 into 4d19b967d3994aac8a7c0747531c96ec(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:42,521 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:42,521 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/A, priority=13, startTime=1733834142503; duration=0sec 2024-12-10T12:35:42,521 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:42,521 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:A 2024-12-10T12:35:42,522 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:42,522 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:42,522 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/C is initiating minor compaction (all files) 2024-12-10T12:35:42,522 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/C in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,522 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/2a43868c6a1840308c4bb8dd67309ba1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/a2d854b2c02c49e780f2ad0496291a5b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/3812502b39f84d7887ba015f2e425dab] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=35.5 K 2024-12-10T12:35:42,523 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2a43868c6a1840308c4bb8dd67309ba1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733834137337 2024-12-10T12:35:42,523 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting a2d854b2c02c49e780f2ad0496291a5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733834138470 2024-12-10T12:35:42,523 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3812502b39f84d7887ba015f2e425dab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733834140604 2024-12-10T12:35:42,527 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#C#compaction#365 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:42,528 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/5692e96f973447cc9019412e569d12ac is 50, key is test_row_0/C:col10/1733834141214/Put/seqid=0 2024-12-10T12:35:42,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742259_1435 (size=12459) 2024-12-10T12:35:42,615 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:42,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T12:35:42,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:42,615 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-10T12:35:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:42,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/36ba2594a1db4869b02f821d3338f315 is 50, key is test_row_0/A:col10/1733834141238/Put/seqid=0 2024-12-10T12:35:42,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742260_1436 (size=12151) 2024-12-10T12:35:42,917 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/ca5e4126f8404ea69f79ed0bab6fe34b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/ca5e4126f8404ea69f79ed0bab6fe34b 2024-12-10T12:35:42,920 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/B of 19609dde5fb8a6633d70503eaf3922b6 into ca5e4126f8404ea69f79ed0bab6fe34b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:42,920 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:42,920 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/B, priority=13, startTime=1733834142503; duration=0sec 2024-12-10T12:35:42,920 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:42,920 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:B 2024-12-10T12:35:42,935 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/5692e96f973447cc9019412e569d12ac as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5692e96f973447cc9019412e569d12ac 2024-12-10T12:35:42,939 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/C of 19609dde5fb8a6633d70503eaf3922b6 into 5692e96f973447cc9019412e569d12ac(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:42,939 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:42,939 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/C, priority=13, startTime=1733834142503; duration=0sec 2024-12-10T12:35:42,939 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:42,939 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:C 2024-12-10T12:35:43,023 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/36ba2594a1db4869b02f821d3338f315 2024-12-10T12:35:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/8a357c5946bd4446be0ff784950d638e is 50, key is test_row_0/B:col10/1733834141238/Put/seqid=0 2024-12-10T12:35:43,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742261_1437 (size=12151) 2024-12-10T12:35:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-12-10T12:35:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:43,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:43,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834203371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834203374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834203374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834203375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834203375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,432 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/8a357c5946bd4446be0ff784950d638e 2024-12-10T12:35:43,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/71f143353477468da2d9645bf17aa614 is 50, key is test_row_0/C:col10/1733834141238/Put/seqid=0 2024-12-10T12:35:43,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742262_1438 (size=12151) 2024-12-10T12:35:43,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834203476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834203478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834203479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834203479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834203479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834203681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834203681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834203682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834203682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834203682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:43,841 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/71f143353477468da2d9645bf17aa614 2024-12-10T12:35:43,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/36ba2594a1db4869b02f821d3338f315 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/36ba2594a1db4869b02f821d3338f315 2024-12-10T12:35:43,847 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/36ba2594a1db4869b02f821d3338f315, entries=150, sequenceid=154, filesize=11.9 K 2024-12-10T12:35:43,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/8a357c5946bd4446be0ff784950d638e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/8a357c5946bd4446be0ff784950d638e 2024-12-10T12:35:43,860 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/8a357c5946bd4446be0ff784950d638e, entries=150, sequenceid=154, filesize=11.9 K 2024-12-10T12:35:43,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/71f143353477468da2d9645bf17aa614 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/71f143353477468da2d9645bf17aa614 2024-12-10T12:35:43,864 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/71f143353477468da2d9645bf17aa614, entries=150, sequenceid=154, filesize=11.9 K 2024-12-10T12:35:43,864 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 19609dde5fb8a6633d70503eaf3922b6 in 1249ms, sequenceid=154, compaction requested=false 2024-12-10T12:35:43,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:43,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:43,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=135 2024-12-10T12:35:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=135 2024-12-10T12:35:43,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=135, resume processing ppid=134 2024-12-10T12:35:43,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=135, ppid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6190 sec 2024-12-10T12:35:43,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 2.6230 sec 2024-12-10T12:35:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:43,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-10T12:35:43,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:43,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:43,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:43,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:43,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:43,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:43,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/f7d644e757924ae7b846a185776cb218 is 50, key is test_row_0/A:col10/1733834143989/Put/seqid=0 2024-12-10T12:35:44,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742263_1439 (size=19321) 2024-12-10T12:35:44,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834204026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834204030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834204031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834204031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834204032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834204133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834204137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834204140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834204140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834204140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834204336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834204341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834204343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834204344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834204345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/f7d644e757924ae7b846a185776cb218 2024-12-10T12:35:44,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/499aee31be594624bd2f0b8448172475 is 50, key is test_row_0/B:col10/1733834143989/Put/seqid=0 2024-12-10T12:35:44,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742264_1440 (size=12151) 2024-12-10T12:35:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834204641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834204645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834204647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834204649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834204649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:44,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/499aee31be594624bd2f0b8448172475 2024-12-10T12:35:44,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ac29556b95aa48dcae17dbd79009cc4d is 50, key is test_row_0/C:col10/1733834143989/Put/seqid=0 2024-12-10T12:35:44,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742265_1441 (size=12151) 2024-12-10T12:35:44,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ac29556b95aa48dcae17dbd79009cc4d 2024-12-10T12:35:44,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/f7d644e757924ae7b846a185776cb218 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/f7d644e757924ae7b846a185776cb218 2024-12-10T12:35:44,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/f7d644e757924ae7b846a185776cb218, entries=300, sequenceid=176, filesize=18.9 K 2024-12-10T12:35:44,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/499aee31be594624bd2f0b8448172475 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/499aee31be594624bd2f0b8448172475 2024-12-10T12:35:44,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/499aee31be594624bd2f0b8448172475, entries=150, sequenceid=176, filesize=11.9 K 2024-12-10T12:35:44,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ac29556b95aa48dcae17dbd79009cc4d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ac29556b95aa48dcae17dbd79009cc4d 2024-12-10T12:35:44,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ac29556b95aa48dcae17dbd79009cc4d, entries=150, sequenceid=176, filesize=11.9 K 2024-12-10T12:35:44,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 19609dde5fb8a6633d70503eaf3922b6 in 854ms, sequenceid=176, compaction requested=true 2024-12-10T12:35:44,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:44,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:44,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:44,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:44,845 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:44,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:44,845 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:44,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:44,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:44,853 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:44,853 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:44,853 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/B is initiating minor compaction (all files) 2024-12-10T12:35:44,853 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/A is initiating minor compaction (all files) 2024-12-10T12:35:44,853 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/A in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:44,853 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/B in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:44,854 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/4d19b967d3994aac8a7c0747531c96ec, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/36ba2594a1db4869b02f821d3338f315, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/f7d644e757924ae7b846a185776cb218] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=42.9 K 2024-12-10T12:35:44,854 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/ca5e4126f8404ea69f79ed0bab6fe34b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/8a357c5946bd4446be0ff784950d638e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/499aee31be594624bd2f0b8448172475] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=35.9 K 2024-12-10T12:35:44,854 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4d19b967d3994aac8a7c0747531c96ec, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733834140604 2024-12-10T12:35:44,854 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ca5e4126f8404ea69f79ed0bab6fe34b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733834140604 2024-12-10T12:35:44,854 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 36ba2594a1db4869b02f821d3338f315, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733834141233 2024-12-10T12:35:44,854 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 8a357c5946bd4446be0ff784950d638e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733834141233 2024-12-10T12:35:44,854 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f7d644e757924ae7b846a185776cb218, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733834143369 2024-12-10T12:35:44,854 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 499aee31be594624bd2f0b8448172475, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733834143989 2024-12-10T12:35:44,860 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#B#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:44,860 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#A#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:44,860 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/2e2af518f621487b9b344f1d768f48ea is 50, key is test_row_0/B:col10/1733834143989/Put/seqid=0 2024-12-10T12:35:44,860 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/b837ed786e5945e0a5f076e573d549ff is 50, key is test_row_0/A:col10/1733834143989/Put/seqid=0 2024-12-10T12:35:44,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742266_1442 (size=12561) 2024-12-10T12:35:44,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742267_1443 (size=12561) 2024-12-10T12:35:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:45,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-10T12:35:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:45,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:45,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/46828cc12e2c41dcad1d3bdd2b3ba51d is 50, key is test_row_0/A:col10/1733834144029/Put/seqid=0 2024-12-10T12:35:45,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742268_1444 (size=14541) 2024-12-10T12:35:45,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/46828cc12e2c41dcad1d3bdd2b3ba51d 2024-12-10T12:35:45,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/21eea7ebdd0a4115815b34a2961e28f2 is 50, key is test_row_0/B:col10/1733834144029/Put/seqid=0 2024-12-10T12:35:45,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742269_1445 (size=12151) 2024-12-10T12:35:45,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834205168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834205169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834205170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834205171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834205172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,267 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/b837ed786e5945e0a5f076e573d549ff as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/b837ed786e5945e0a5f076e573d549ff 2024-12-10T12:35:45,267 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/2e2af518f621487b9b344f1d768f48ea as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/2e2af518f621487b9b344f1d768f48ea 2024-12-10T12:35:45,270 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/B of 19609dde5fb8a6633d70503eaf3922b6 into 2e2af518f621487b9b344f1d768f48ea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:45,270 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/A of 19609dde5fb8a6633d70503eaf3922b6 into b837ed786e5945e0a5f076e573d549ff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:45,271 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:45,271 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:45,271 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/A, priority=13, startTime=1733834144845; duration=0sec 2024-12-10T12:35:45,271 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/B, priority=13, startTime=1733834144845; duration=0sec 2024-12-10T12:35:45,271 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:45,271 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:45,271 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:B 2024-12-10T12:35:45,271 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:A 2024-12-10T12:35:45,271 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:45,271 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:45,271 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/C is initiating minor compaction (all files) 2024-12-10T12:35:45,272 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/C in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:45,272 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5692e96f973447cc9019412e569d12ac, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/71f143353477468da2d9645bf17aa614, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ac29556b95aa48dcae17dbd79009cc4d] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=35.9 K 2024-12-10T12:35:45,272 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5692e96f973447cc9019412e569d12ac, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733834140604 2024-12-10T12:35:45,272 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 71f143353477468da2d9645bf17aa614, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733834141233 2024-12-10T12:35:45,272 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting ac29556b95aa48dcae17dbd79009cc4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733834143989 2024-12-10T12:35:45,278 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#C#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:45,278 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/4242c866b1624ee9bb355043c08fea04 is 50, key is test_row_0/C:col10/1733834143989/Put/seqid=0 2024-12-10T12:35:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834205275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834205275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834205276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742270_1446 (size=12561) 2024-12-10T12:35:45,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834205278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834205278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-12-10T12:35:45,349 INFO [Thread-1832 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 134 completed 2024-12-10T12:35:45,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-12-10T12:35:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-12-10T12:35:45,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:45,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:45,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-12-10T12:35:45,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834205480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834205480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834205481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834205484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834205484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,502 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:45,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T12:35:45,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:45,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:45,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:45,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/21eea7ebdd0a4115815b34a2961e28f2 2024-12-10T12:35:45,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/374696d35afc4aaca76f081dfd74d305 is 50, key is test_row_0/C:col10/1733834144029/Put/seqid=0 2024-12-10T12:35:45,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742271_1447 (size=12151) 2024-12-10T12:35:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-12-10T12:35:45,654 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:45,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T12:35:45,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:45,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:45,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:45,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,685 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/4242c866b1624ee9bb355043c08fea04 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/4242c866b1624ee9bb355043c08fea04 2024-12-10T12:35:45,688 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/C of 19609dde5fb8a6633d70503eaf3922b6 into 4242c866b1624ee9bb355043c08fea04(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:45,688 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:45,688 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/C, priority=13, startTime=1733834144845; duration=0sec 2024-12-10T12:35:45,688 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:45,688 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:C 2024-12-10T12:35:45,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834205787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834205787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834205787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834205788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834205789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:45,807 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:45,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T12:35:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:45,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-12-10T12:35:45,959 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:45,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T12:35:45,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:45,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:45,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:45,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:45,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/374696d35afc4aaca76f081dfd74d305 2024-12-10T12:35:45,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/46828cc12e2c41dcad1d3bdd2b3ba51d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/46828cc12e2c41dcad1d3bdd2b3ba51d 2024-12-10T12:35:45,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/46828cc12e2c41dcad1d3bdd2b3ba51d, entries=200, sequenceid=193, filesize=14.2 K 2024-12-10T12:35:45,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/21eea7ebdd0a4115815b34a2961e28f2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/21eea7ebdd0a4115815b34a2961e28f2 2024-12-10T12:35:45,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/21eea7ebdd0a4115815b34a2961e28f2, entries=150, sequenceid=193, filesize=11.9 K 2024-12-10T12:35:45,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/374696d35afc4aaca76f081dfd74d305 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/374696d35afc4aaca76f081dfd74d305 2024-12-10T12:35:45,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/374696d35afc4aaca76f081dfd74d305, entries=150, sequenceid=193, filesize=11.9 K 2024-12-10T12:35:45,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 19609dde5fb8a6633d70503eaf3922b6 in 847ms, sequenceid=193, compaction requested=false 2024-12-10T12:35:45,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:46,111 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:46,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T12:35:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:46,112 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-10T12:35:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/afb921e4e35740b69c112bd2a902b137 is 50, key is test_row_0/A:col10/1733834145168/Put/seqid=0 2024-12-10T12:35:46,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742272_1448 (size=12151) 2024-12-10T12:35:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:46,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834206330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834206331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834206331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834206332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834206333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834206435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834206438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834206438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834206438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834206438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-12-10T12:35:46,521 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/afb921e4e35740b69c112bd2a902b137 2024-12-10T12:35:46,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/0bc48d59b66c495ca9bbf60f0b367e7f is 50, key is test_row_0/B:col10/1733834145168/Put/seqid=0 2024-12-10T12:35:46,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742273_1449 (size=12151) 2024-12-10T12:35:46,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834206638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834206641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834206642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834206642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834206642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,930 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/0bc48d59b66c495ca9bbf60f0b367e7f 2024-12-10T12:35:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/c01927583a064fadb29c77836e4985fa is 50, key is test_row_0/C:col10/1733834145168/Put/seqid=0 2024-12-10T12:35:46,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742274_1450 (size=12151) 2024-12-10T12:35:46,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834206943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834206944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834206946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834206947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:46,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834206948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,339 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/c01927583a064fadb29c77836e4985fa 2024-12-10T12:35:47,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/afb921e4e35740b69c112bd2a902b137 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/afb921e4e35740b69c112bd2a902b137 2024-12-10T12:35:47,399 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/afb921e4e35740b69c112bd2a902b137, entries=150, sequenceid=215, filesize=11.9 K 2024-12-10T12:35:47,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/0bc48d59b66c495ca9bbf60f0b367e7f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/0bc48d59b66c495ca9bbf60f0b367e7f 2024-12-10T12:35:47,402 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/0bc48d59b66c495ca9bbf60f0b367e7f, entries=150, sequenceid=215, filesize=11.9 K 2024-12-10T12:35:47,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/c01927583a064fadb29c77836e4985fa as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/c01927583a064fadb29c77836e4985fa 2024-12-10T12:35:47,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/c01927583a064fadb29c77836e4985fa, entries=150, sequenceid=215, filesize=11.9 K 2024-12-10T12:35:47,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 19609dde5fb8a6633d70503eaf3922b6 in 1293ms, sequenceid=215, compaction requested=true 2024-12-10T12:35:47,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:47,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:47,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-12-10T12:35:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=137 2024-12-10T12:35:47,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=137, resume processing ppid=136 2024-12-10T12:35:47,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=137, ppid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0550 sec 2024-12-10T12:35:47,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 2.0580 sec 2024-12-10T12:35:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:47,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-10T12:35:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:47,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/27e08c1968c84ed7a1b3438c7f77ba21 is 50, key is test_row_0/A:col10/1733834146332/Put/seqid=0 2024-12-10T12:35:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-12-10T12:35:47,454 INFO [Thread-1832 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 136 completed 2024-12-10T12:35:47,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:47,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742275_1451 (size=14541) 2024-12-10T12:35:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-12-10T12:35:47,456 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-12-10T12:35:47,456 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:47,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:47,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834207463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834207462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834207464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834207464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834207468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-12-10T12:35:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834207569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834207569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834207569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834207569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834207571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,608 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:47,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T12:35:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:47,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-12-10T12:35:47,760 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:47,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T12:35:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:47,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:47,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:47,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834207772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834207773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834207773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834207774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834207775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:47,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/27e08c1968c84ed7a1b3438c7f77ba21 2024-12-10T12:35:47,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/814fddcbaac0405e9d0b0b61b7433af2 is 50, key is test_row_0/B:col10/1733834146332/Put/seqid=0 2024-12-10T12:35:47,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742276_1452 (size=12151) 2024-12-10T12:35:47,912 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:47,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T12:35:47,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:47,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:47,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:47,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:47,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-12-10T12:35:48,064 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:48,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T12:35:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834208075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834208077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834208077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834208080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834208080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,216 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:48,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T12:35:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/814fddcbaac0405e9d0b0b61b7433af2 2024-12-10T12:35:48,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/21dce06a32bb4de682786f2979077dd7 is 50, key is test_row_0/C:col10/1733834146332/Put/seqid=0 2024-12-10T12:35:48,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742277_1453 (size=12151) 2024-12-10T12:35:48,369 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:48,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T12:35:48,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:48,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,521 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:48,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T12:35:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-12-10T12:35:48,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834208581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834208582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834208583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834208585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834208586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:48,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/21dce06a32bb4de682786f2979077dd7 2024-12-10T12:35:48,674 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:48,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T12:35:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:48,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/27e08c1968c84ed7a1b3438c7f77ba21 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/27e08c1968c84ed7a1b3438c7f77ba21 2024-12-10T12:35:48,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/27e08c1968c84ed7a1b3438c7f77ba21, entries=200, sequenceid=233, filesize=14.2 K 2024-12-10T12:35:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/814fddcbaac0405e9d0b0b61b7433af2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/814fddcbaac0405e9d0b0b61b7433af2 2024-12-10T12:35:48,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/814fddcbaac0405e9d0b0b61b7433af2, entries=150, sequenceid=233, filesize=11.9 K 2024-12-10T12:35:48,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/21dce06a32bb4de682786f2979077dd7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/21dce06a32bb4de682786f2979077dd7 2024-12-10T12:35:48,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/21dce06a32bb4de682786f2979077dd7, entries=150, sequenceid=233, filesize=11.9 K 2024-12-10T12:35:48,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 19609dde5fb8a6633d70503eaf3922b6 in 1239ms, sequenceid=233, compaction requested=true 2024-12-10T12:35:48,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:48,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:48,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:48,686 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:35:48,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:48,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:48,686 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:35:48,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:48,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:48,687 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:35:48,687 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:35:48,687 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/A is initiating minor compaction (all files) 2024-12-10T12:35:48,687 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/B is initiating minor compaction (all files) 2024-12-10T12:35:48,687 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/B in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,687 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/A in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,687 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/2e2af518f621487b9b344f1d768f48ea, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/21eea7ebdd0a4115815b34a2961e28f2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/0bc48d59b66c495ca9bbf60f0b367e7f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/814fddcbaac0405e9d0b0b61b7433af2] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=47.9 K 2024-12-10T12:35:48,687 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/b837ed786e5945e0a5f076e573d549ff, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/46828cc12e2c41dcad1d3bdd2b3ba51d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/afb921e4e35740b69c112bd2a902b137, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/27e08c1968c84ed7a1b3438c7f77ba21] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=52.5 K 2024-12-10T12:35:48,687 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2e2af518f621487b9b344f1d768f48ea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733834143989 2024-12-10T12:35:48,687 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting b837ed786e5945e0a5f076e573d549ff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733834143989 2024-12-10T12:35:48,687 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 21eea7ebdd0a4115815b34a2961e28f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733834144029 2024-12-10T12:35:48,688 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 46828cc12e2c41dcad1d3bdd2b3ba51d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733834144029 2024-12-10T12:35:48,688 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 0bc48d59b66c495ca9bbf60f0b367e7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733834145168 2024-12-10T12:35:48,688 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting afb921e4e35740b69c112bd2a902b137, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733834145168 2024-12-10T12:35:48,688 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 814fddcbaac0405e9d0b0b61b7433af2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733834146331 2024-12-10T12:35:48,688 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 27e08c1968c84ed7a1b3438c7f77ba21, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733834146326 2024-12-10T12:35:48,695 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:48,695 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/044ac32adde74d02b070d374d66cdbe6 is 50, key is test_row_0/B:col10/1733834146332/Put/seqid=0 2024-12-10T12:35:48,703 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:48,703 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/9c91463a2f304963a6b9eb8c225c9751 is 50, key is test_row_0/A:col10/1733834146332/Put/seqid=0 2024-12-10T12:35:48,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742279_1455 (size=12697) 2024-12-10T12:35:48,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742278_1454 (size=12697) 2024-12-10T12:35:48,826 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:48,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T12:35:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:48,826 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-10T12:35:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:48,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/7243db1f3f5d472899ad50824b562843 is 50, key is test_row_0/A:col10/1733834147457/Put/seqid=0 2024-12-10T12:35:48,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742280_1456 (size=12151) 2024-12-10T12:35:49,113 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/9c91463a2f304963a6b9eb8c225c9751 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9c91463a2f304963a6b9eb8c225c9751 2024-12-10T12:35:49,116 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/A of 19609dde5fb8a6633d70503eaf3922b6 into 9c91463a2f304963a6b9eb8c225c9751(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:49,116 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:49,116 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/A, priority=12, startTime=1733834148686; duration=0sec 2024-12-10T12:35:49,117 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:49,117 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:A 2024-12-10T12:35:49,117 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:35:49,118 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:35:49,118 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/C is initiating minor compaction (all files) 2024-12-10T12:35:49,118 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/C in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:49,118 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/4242c866b1624ee9bb355043c08fea04, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/374696d35afc4aaca76f081dfd74d305, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/c01927583a064fadb29c77836e4985fa, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/21dce06a32bb4de682786f2979077dd7] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=47.9 K 2024-12-10T12:35:49,119 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4242c866b1624ee9bb355043c08fea04, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733834143989 2024-12-10T12:35:49,119 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 374696d35afc4aaca76f081dfd74d305, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733834144029 2024-12-10T12:35:49,119 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting c01927583a064fadb29c77836e4985fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733834145168 2024-12-10T12:35:49,119 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 21dce06a32bb4de682786f2979077dd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733834146331 2024-12-10T12:35:49,120 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/044ac32adde74d02b070d374d66cdbe6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/044ac32adde74d02b070d374d66cdbe6 2024-12-10T12:35:49,124 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/B of 19609dde5fb8a6633d70503eaf3922b6 into 044ac32adde74d02b070d374d66cdbe6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:49,124 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:49,124 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/B, priority=12, startTime=1733834148686; duration=0sec 2024-12-10T12:35:49,124 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:49,124 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:B 2024-12-10T12:35:49,127 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#C#compaction#387 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:49,127 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/223ff2340c704e5eaab06fd51f7bb185 is 50, key is test_row_0/C:col10/1733834146332/Put/seqid=0 2024-12-10T12:35:49,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742281_1457 (size=12697) 2024-12-10T12:35:49,141 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/223ff2340c704e5eaab06fd51f7bb185 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/223ff2340c704e5eaab06fd51f7bb185 2024-12-10T12:35:49,146 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/C of 19609dde5fb8a6633d70503eaf3922b6 into 223ff2340c704e5eaab06fd51f7bb185(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:49,146 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:49,146 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/C, priority=12, startTime=1733834148686; duration=0sec 2024-12-10T12:35:49,146 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:49,146 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:C 2024-12-10T12:35:49,233 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/7243db1f3f5d472899ad50824b562843 2024-12-10T12:35:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/3b4a9b4526f14b51a7fd363dba61e2fb is 50, key is test_row_0/B:col10/1733834147457/Put/seqid=0 2024-12-10T12:35:49,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742282_1458 (size=12151) 2024-12-10T12:35:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-12-10T12:35:49,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:49,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834209602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834209603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834209604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834209607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834209608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,646 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/3b4a9b4526f14b51a7fd363dba61e2fb 2024-12-10T12:35:49,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ec27480385be4843a8feefbd1e281820 is 50, key is test_row_0/C:col10/1733834147457/Put/seqid=0 2024-12-10T12:35:49,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742283_1459 (size=12151) 2024-12-10T12:35:49,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834209709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834209709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834209709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834209712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834209712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834209911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834209912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834209912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834209916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834209916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,055 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ec27480385be4843a8feefbd1e281820 2024-12-10T12:35:50,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/7243db1f3f5d472899ad50824b562843 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/7243db1f3f5d472899ad50824b562843 2024-12-10T12:35:50,061 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/7243db1f3f5d472899ad50824b562843, entries=150, sequenceid=251, filesize=11.9 K 2024-12-10T12:35:50,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/3b4a9b4526f14b51a7fd363dba61e2fb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/3b4a9b4526f14b51a7fd363dba61e2fb 2024-12-10T12:35:50,065 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/3b4a9b4526f14b51a7fd363dba61e2fb, entries=150, sequenceid=251, filesize=11.9 K 2024-12-10T12:35:50,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ec27480385be4843a8feefbd1e281820 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ec27480385be4843a8feefbd1e281820 2024-12-10T12:35:50,068 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ec27480385be4843a8feefbd1e281820, entries=150, sequenceid=251, filesize=11.9 K 2024-12-10T12:35:50,069 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 19609dde5fb8a6633d70503eaf3922b6 in 1243ms, sequenceid=251, compaction requested=false 2024-12-10T12:35:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-12-10T12:35:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=139 2024-12-10T12:35:50,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=139, resume processing ppid=138 2024-12-10T12:35:50,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=139, ppid=138, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6140 sec 2024-12-10T12:35:50,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=138, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 2.6160 sec 2024-12-10T12:35:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:50,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-10T12:35:50,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:50,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:50,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:50,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:50,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:50,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:50,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/eefba4d0edba4588bf614266ee515696 is 50, key is test_row_0/A:col10/1733834149602/Put/seqid=0 2024-12-10T12:35:50,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742284_1460 (size=12301) 2024-12-10T12:35:50,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834210230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834210230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834210231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834210232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834210233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834210334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834210339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834210339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834210340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834210341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834210536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834210543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834210543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834210544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834210544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/eefba4d0edba4588bf614266ee515696 2024-12-10T12:35:50,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/dbdc0b2af9f94229873e873aef8d4bf0 is 50, key is test_row_0/B:col10/1733834149602/Put/seqid=0 2024-12-10T12:35:50,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742285_1461 (size=12301) 2024-12-10T12:35:50,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834210840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834210848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834210849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834210849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:50,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834210849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:51,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/dbdc0b2af9f94229873e873aef8d4bf0 2024-12-10T12:35:51,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/153bc74088a54d609c4f28b19aa58a60 is 50, key is test_row_0/C:col10/1733834149602/Put/seqid=0 2024-12-10T12:35:51,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742286_1462 (size=12301) 2024-12-10T12:35:51,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834211345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834211352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834211354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834211354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:51,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834211356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:51,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/153bc74088a54d609c4f28b19aa58a60 2024-12-10T12:35:51,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/eefba4d0edba4588bf614266ee515696 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/eefba4d0edba4588bf614266ee515696 2024-12-10T12:35:51,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/eefba4d0edba4588bf614266ee515696, entries=150, sequenceid=273, filesize=12.0 K 2024-12-10T12:35:51,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/dbdc0b2af9f94229873e873aef8d4bf0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dbdc0b2af9f94229873e873aef8d4bf0 2024-12-10T12:35:51,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dbdc0b2af9f94229873e873aef8d4bf0, entries=150, sequenceid=273, filesize=12.0 K 2024-12-10T12:35:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/153bc74088a54d609c4f28b19aa58a60 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/153bc74088a54d609c4f28b19aa58a60 2024-12-10T12:35:51,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/153bc74088a54d609c4f28b19aa58a60, entries=150, sequenceid=273, filesize=12.0 K 2024-12-10T12:35:51,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 19609dde5fb8a6633d70503eaf3922b6 in 1244ms, sequenceid=273, compaction requested=true 2024-12-10T12:35:51,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:51,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:51,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:51,462 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:51,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:51,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:51,462 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:51,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:51,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:51,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:51,463 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:51,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/B is initiating minor compaction (all files) 2024-12-10T12:35:51,463 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/A is initiating minor compaction (all files) 2024-12-10T12:35:51,463 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/A in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:51,463 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/B in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:51,463 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/044ac32adde74d02b070d374d66cdbe6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/3b4a9b4526f14b51a7fd363dba61e2fb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dbdc0b2af9f94229873e873aef8d4bf0] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=36.3 K 2024-12-10T12:35:51,463 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9c91463a2f304963a6b9eb8c225c9751, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/7243db1f3f5d472899ad50824b562843, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/eefba4d0edba4588bf614266ee515696] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=36.3 K 2024-12-10T12:35:51,463 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9c91463a2f304963a6b9eb8c225c9751, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733834146331 2024-12-10T12:35:51,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 044ac32adde74d02b070d374d66cdbe6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733834146331 2024-12-10T12:35:51,463 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7243db1f3f5d472899ad50824b562843, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733834147457 2024-12-10T12:35:51,463 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 3b4a9b4526f14b51a7fd363dba61e2fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733834147457 2024-12-10T12:35:51,464 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting eefba4d0edba4588bf614266ee515696, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733834149600 2024-12-10T12:35:51,464 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting dbdc0b2af9f94229873e873aef8d4bf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733834149600 2024-12-10T12:35:51,469 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#B#compaction#393 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:51,469 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:51,469 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/cfa03c539edd4451b13b0298c47f3dd7 is 50, key is test_row_0/B:col10/1733834149602/Put/seqid=0 2024-12-10T12:35:51,469 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/27394bfb2f23415595543d9af3f9c7f1 is 50, key is test_row_0/A:col10/1733834149602/Put/seqid=0 2024-12-10T12:35:51,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742287_1463 (size=12949) 2024-12-10T12:35:51,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742288_1464 (size=12949) 2024-12-10T12:35:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-12-10T12:35:51,560 INFO [Thread-1832 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 138 completed 2024-12-10T12:35:51,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-12-10T12:35:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=140 2024-12-10T12:35:51,562 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:51,563 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:51,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=140 2024-12-10T12:35:51,714 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:51,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-10T12:35:51,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:51,714 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-10T12:35:51,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:51,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:51,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:51,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:51,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:51,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/e8f99b602055457a995f442db2a819ec is 50, key is test_row_0/A:col10/1733834150229/Put/seqid=0 2024-12-10T12:35:51,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742289_1465 (size=12301) 2024-12-10T12:35:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=140 2024-12-10T12:35:51,880 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/cfa03c539edd4451b13b0298c47f3dd7 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/cfa03c539edd4451b13b0298c47f3dd7 2024-12-10T12:35:51,880 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/27394bfb2f23415595543d9af3f9c7f1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/27394bfb2f23415595543d9af3f9c7f1 2024-12-10T12:35:51,886 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/B of 19609dde5fb8a6633d70503eaf3922b6 into cfa03c539edd4451b13b0298c47f3dd7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:51,886 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/A of 19609dde5fb8a6633d70503eaf3922b6 into 27394bfb2f23415595543d9af3f9c7f1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:51,886 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:51,886 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:51,886 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/B, priority=13, startTime=1733834151462; duration=0sec 2024-12-10T12:35:51,886 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/A, priority=13, startTime=1733834151462; duration=0sec 2024-12-10T12:35:51,886 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:51,886 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:35:51,886 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:A 2024-12-10T12:35:51,886 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:B 2024-12-10T12:35:51,886 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:35:51,887 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:35:51,887 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): 19609dde5fb8a6633d70503eaf3922b6/C is initiating minor compaction (all files) 2024-12-10T12:35:51,887 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 19609dde5fb8a6633d70503eaf3922b6/C in TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:51,887 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/223ff2340c704e5eaab06fd51f7bb185, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ec27480385be4843a8feefbd1e281820, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/153bc74088a54d609c4f28b19aa58a60] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp, totalSize=36.3 K 2024-12-10T12:35:51,888 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 223ff2340c704e5eaab06fd51f7bb185, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733834146331 2024-12-10T12:35:51,888 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting ec27480385be4843a8feefbd1e281820, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733834147457 2024-12-10T12:35:51,888 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 153bc74088a54d609c4f28b19aa58a60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733834149600 2024-12-10T12:35:51,893 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 19609dde5fb8a6633d70503eaf3922b6#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:35:51,894 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/773d87d43eb14d9c9c05a730313e7494 is 50, key is test_row_0/C:col10/1733834149602/Put/seqid=0 2024-12-10T12:35:51,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742290_1466 (size=12949) 2024-12-10T12:35:52,121 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/e8f99b602055457a995f442db2a819ec 2024-12-10T12:35:52,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/d831b6e26715466abcc0747a483e24a3 is 50, key is test_row_0/B:col10/1733834150229/Put/seqid=0 2024-12-10T12:35:52,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742291_1467 (size=12301) 2024-12-10T12:35:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=140 2024-12-10T12:35:52,304 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/773d87d43eb14d9c9c05a730313e7494 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/773d87d43eb14d9c9c05a730313e7494 2024-12-10T12:35:52,308 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 19609dde5fb8a6633d70503eaf3922b6/C of 19609dde5fb8a6633d70503eaf3922b6 into 773d87d43eb14d9c9c05a730313e7494(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:35:52,308 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:52,308 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6., storeName=19609dde5fb8a6633d70503eaf3922b6/C, priority=13, startTime=1733834151462; duration=0sec 2024-12-10T12:35:52,308 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:52,308 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:C 2024-12-10T12:35:52,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. as already flushing 2024-12-10T12:35:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:52,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834212372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834212372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834212373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834212374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834212376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834212477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834212477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834212477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834212478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834212481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,533 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/d831b6e26715466abcc0747a483e24a3 2024-12-10T12:35:52,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ed15365bbaf740ad92e8353ee0ce6d12 is 50, key is test_row_0/C:col10/1733834150229/Put/seqid=0 2024-12-10T12:35:52,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742292_1468 (size=12301) 2024-12-10T12:35:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=140 2024-12-10T12:35:52,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39578 deadline: 1733834212679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39646 deadline: 1733834212680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39602 deadline: 1733834212680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1733834212682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39588 deadline: 1733834212685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 2024-12-10T12:35:52,926 DEBUG [Thread-1835 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0f2423f3 to 127.0.0.1:64063 2024-12-10T12:35:52,926 DEBUG [Thread-1835 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,926 DEBUG [Thread-1837 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x184771cf to 127.0.0.1:64063 2024-12-10T12:35:52,926 DEBUG [Thread-1837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,927 DEBUG [Thread-1839 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x076f0408 to 127.0.0.1:64063 2024-12-10T12:35:52,927 DEBUG [Thread-1839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,929 DEBUG [Thread-1833 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x41b0e7b6 to 127.0.0.1:64063 2024-12-10T12:35:52,929 DEBUG [Thread-1833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,929 DEBUG [Thread-1841 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0c692575 to 127.0.0.1:64063 2024-12-10T12:35:52,930 DEBUG [Thread-1841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,942 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ed15365bbaf740ad92e8353ee0ce6d12 2024-12-10T12:35:52,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/e8f99b602055457a995f442db2a819ec as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/e8f99b602055457a995f442db2a819ec 2024-12-10T12:35:52,949 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/e8f99b602055457a995f442db2a819ec, entries=150, sequenceid=290, filesize=12.0 K 2024-12-10T12:35:52,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/d831b6e26715466abcc0747a483e24a3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/d831b6e26715466abcc0747a483e24a3 2024-12-10T12:35:52,953 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/d831b6e26715466abcc0747a483e24a3, entries=150, sequenceid=290, filesize=12.0 K 2024-12-10T12:35:52,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ed15365bbaf740ad92e8353ee0ce6d12 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ed15365bbaf740ad92e8353ee0ce6d12 2024-12-10T12:35:52,957 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ed15365bbaf740ad92e8353ee0ce6d12, entries=150, sequenceid=290, filesize=12.0 K 2024-12-10T12:35:52,958 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 19609dde5fb8a6633d70503eaf3922b6 in 1244ms, sequenceid=290, compaction requested=false 2024-12-10T12:35:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-12-10T12:35:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=141 2024-12-10T12:35:52,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=141, resume processing ppid=140 2024-12-10T12:35:52,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=141, ppid=140, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-12-10T12:35:52,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=140, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.4000 sec 2024-12-10T12:35:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:52,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-10T12:35:52,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:52,984 DEBUG [Thread-1830 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d672ed2 to 127.0.0.1:64063 2024-12-10T12:35:52,985 DEBUG [Thread-1830 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:52,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:52,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:52,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:52,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:52,987 DEBUG [Thread-1824 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7043f683 to 127.0.0.1:64063 2024-12-10T12:35:52,987 DEBUG [Thread-1824 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,987 DEBUG [Thread-1828 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x34b30c39 to 127.0.0.1:64063 2024-12-10T12:35:52,987 DEBUG [Thread-1828 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,990 DEBUG [Thread-1822 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x61d38088 to 127.0.0.1:64063 2024-12-10T12:35:52,991 DEBUG [Thread-1822 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/0073169fda60446da5cb3b6a9fd7f86e is 50, key is test_row_0/A:col10/1733834152982/Put/seqid=0 2024-12-10T12:35:52,992 DEBUG [Thread-1826 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b0c2472 to 127.0.0.1:64063 2024-12-10T12:35:52,992 DEBUG [Thread-1826 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:52,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742293_1469 (size=12301) 2024-12-10T12:35:53,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/0073169fda60446da5cb3b6a9fd7f86e 2024-12-10T12:35:53,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/5c77f8a5a4e6479ba0146d9d07581bf3 is 50, key is test_row_0/B:col10/1733834152982/Put/seqid=0 2024-12-10T12:35:53,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742294_1470 (size=12301) 2024-12-10T12:35:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=140 2024-12-10T12:35:53,668 INFO [Thread-1832 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 140 completed 2024-12-10T12:35:53,668 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T12:35:53,668 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-10T12:35:53,668 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-10T12:35:53,668 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-10T12:35:53,668 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-10T12:35:53,668 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-10T12:35:53,668 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3473 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10419 rows 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3465 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10395 rows 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3455 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10365 rows 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3475 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10425 rows 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 3465 2024-12-10T12:35:53,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 10394 rows 2024-12-10T12:35:53,669 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-10T12:35:53,669 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7177efc9 to 127.0.0.1:64063 2024-12-10T12:35:53,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:35:53,674 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-12-10T12:35:53,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T12:35:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=142, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=142 2024-12-10T12:35:53,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834153676"}]},"ts":"1733834153676"} 2024-12-10T12:35:53,677 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T12:35:53,721 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T12:35:53,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T12:35:53,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=19609dde5fb8a6633d70503eaf3922b6, UNASSIGN}] 2024-12-10T12:35:53,726 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=19609dde5fb8a6633d70503eaf3922b6, UNASSIGN 2024-12-10T12:35:53,727 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=144 updating hbase:meta row=19609dde5fb8a6633d70503eaf3922b6, regionState=CLOSING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:53,728 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T12:35:53,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE; CloseRegionProcedure 19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=142 2024-12-10T12:35:53,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/5c77f8a5a4e6479ba0146d9d07581bf3 2024-12-10T12:35:53,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ecbfdd63eb4a4761a3f808332db07b37 is 50, key is test_row_0/C:col10/1733834152982/Put/seqid=0 2024-12-10T12:35:53,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742295_1471 (size=12301) 2024-12-10T12:35:53,880 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:53,881 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] handler.UnassignRegionHandler(124): Close 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:53,881 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-12-10T12:35:53,881 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1681): Closing 19609dde5fb8a6633d70503eaf3922b6, disabling compactions & flushes 2024-12-10T12:35:53,881 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1942): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=142 2024-12-10T12:35:54,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ecbfdd63eb4a4761a3f808332db07b37 2024-12-10T12:35:54,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/0073169fda60446da5cb3b6a9fd7f86e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0073169fda60446da5cb3b6a9fd7f86e 2024-12-10T12:35:54,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0073169fda60446da5cb3b6a9fd7f86e, entries=150, sequenceid=313, filesize=12.0 K 2024-12-10T12:35:54,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/5c77f8a5a4e6479ba0146d9d07581bf3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/5c77f8a5a4e6479ba0146d9d07581bf3 2024-12-10T12:35:54,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/5c77f8a5a4e6479ba0146d9d07581bf3, entries=150, sequenceid=313, filesize=12.0 K 2024-12-10T12:35:54,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/ecbfdd63eb4a4761a3f808332db07b37 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ecbfdd63eb4a4761a3f808332db07b37 2024-12-10T12:35:54,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ecbfdd63eb4a4761a3f808332db07b37, entries=150, sequenceid=313, filesize=12.0 K 2024-12-10T12:35:54,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for 19609dde5fb8a6633d70503eaf3922b6 in 1271ms, sequenceid=313, compaction requested=true 2024-12-10T12:35:54,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:54,255 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:54,255 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:54,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:35:54,255 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. after waiting 0 ms 2024-12-10T12:35:54,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:54,255 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:54,255 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. because compaction request was cancelled 2024-12-10T12:35:54,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:35:54,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:54,255 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:A 2024-12-10T12:35:54,255 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. because compaction request was cancelled 2024-12-10T12:35:54,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 19609dde5fb8a6633d70503eaf3922b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:35:54,255 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(2837): Flushing 19609dde5fb8a6633d70503eaf3922b6 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-10T12:35:54,255 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:B 2024-12-10T12:35:54,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:35:54,255 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. because compaction request was cancelled 2024-12-10T12:35:54,255 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 19609dde5fb8a6633d70503eaf3922b6:C 2024-12-10T12:35:54,255 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=A 2024-12-10T12:35:54,255 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:54,255 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=B 2024-12-10T12:35:54,255 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:54,256 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 19609dde5fb8a6633d70503eaf3922b6, store=C 2024-12-10T12:35:54,256 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:54,259 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/002012b91b0340169f36a72b10544a8b is 50, key is test_row_0/A:col10/1733834152985/Put/seqid=0 2024-12-10T12:35:54,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742296_1472 (size=12301) 2024-12-10T12:35:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=142 2024-12-10T12:35:54,664 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/002012b91b0340169f36a72b10544a8b 2024-12-10T12:35:54,675 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/aa5fad0ad66a40b9bddb4c14dc1b9847 is 50, key is test_row_0/B:col10/1733834152985/Put/seqid=0 2024-12-10T12:35:54,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742297_1473 (size=12301) 2024-12-10T12:35:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=142 2024-12-10T12:35:55,080 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/aa5fad0ad66a40b9bddb4c14dc1b9847 2024-12-10T12:35:55,094 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/b3926379326c4ebfb84167d6c89915a3 is 50, key is test_row_0/C:col10/1733834152985/Put/seqid=0 2024-12-10T12:35:55,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742298_1474 (size=12301) 2024-12-10T12:35:55,500 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/b3926379326c4ebfb84167d6c89915a3 2024-12-10T12:35:55,512 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/A/002012b91b0340169f36a72b10544a8b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/002012b91b0340169f36a72b10544a8b 2024-12-10T12:35:55,517 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/002012b91b0340169f36a72b10544a8b, entries=150, sequenceid=320, filesize=12.0 K 2024-12-10T12:35:55,518 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/B/aa5fad0ad66a40b9bddb4c14dc1b9847 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aa5fad0ad66a40b9bddb4c14dc1b9847 2024-12-10T12:35:55,523 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aa5fad0ad66a40b9bddb4c14dc1b9847, entries=150, sequenceid=320, filesize=12.0 K 2024-12-10T12:35:55,524 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/.tmp/C/b3926379326c4ebfb84167d6c89915a3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3926379326c4ebfb84167d6c89915a3 2024-12-10T12:35:55,528 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3926379326c4ebfb84167d6c89915a3, entries=150, sequenceid=320, filesize=12.0 K 2024-12-10T12:35:55,529 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(3040): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 19609dde5fb8a6633d70503eaf3922b6 in 1274ms, sequenceid=320, compaction requested=true 2024-12-10T12:35:55,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9189aa55fe9f4c2082ee7771ec4afbe7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/d751b87d865e4928b31cf8c6449543e5, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/cdaba4e9f64b4f3f8dbb03aa2634b424, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c249720c102645128f93f519938fe744, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c105a2c296124a1a9f21650a88d4cc14, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0e0982a791764b3197ff10273b36d517, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/dfc2e12a9082496085f9e585ab4b8efa, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/2db61739088745f5ab02f7d3b8804665, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/21b8afca1c96497e8117b5ec7c592ea1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/4d19b967d3994aac8a7c0747531c96ec, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/36ba2594a1db4869b02f821d3338f315, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/f7d644e757924ae7b846a185776cb218, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/b837ed786e5945e0a5f076e573d549ff, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/46828cc12e2c41dcad1d3bdd2b3ba51d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/afb921e4e35740b69c112bd2a902b137, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/27e08c1968c84ed7a1b3438c7f77ba21, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9c91463a2f304963a6b9eb8c225c9751, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/7243db1f3f5d472899ad50824b562843, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/eefba4d0edba4588bf614266ee515696] to archive 2024-12-10T12:35:55,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:35:55,532 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9189aa55fe9f4c2082ee7771ec4afbe7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9189aa55fe9f4c2082ee7771ec4afbe7 2024-12-10T12:35:55,532 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/d751b87d865e4928b31cf8c6449543e5 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/d751b87d865e4928b31cf8c6449543e5 2024-12-10T12:35:55,533 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c249720c102645128f93f519938fe744 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c249720c102645128f93f519938fe744 2024-12-10T12:35:55,533 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/2db61739088745f5ab02f7d3b8804665 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/2db61739088745f5ab02f7d3b8804665 2024-12-10T12:35:55,533 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/dfc2e12a9082496085f9e585ab4b8efa to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/dfc2e12a9082496085f9e585ab4b8efa 2024-12-10T12:35:55,533 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0e0982a791764b3197ff10273b36d517 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0e0982a791764b3197ff10273b36d517 2024-12-10T12:35:55,533 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/cdaba4e9f64b4f3f8dbb03aa2634b424 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/cdaba4e9f64b4f3f8dbb03aa2634b424 2024-12-10T12:35:55,533 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c105a2c296124a1a9f21650a88d4cc14 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/c105a2c296124a1a9f21650a88d4cc14 2024-12-10T12:35:55,536 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/21b8afca1c96497e8117b5ec7c592ea1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/21b8afca1c96497e8117b5ec7c592ea1 2024-12-10T12:35:55,536 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/4d19b967d3994aac8a7c0747531c96ec to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/4d19b967d3994aac8a7c0747531c96ec 2024-12-10T12:35:55,536 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/b837ed786e5945e0a5f076e573d549ff to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/b837ed786e5945e0a5f076e573d549ff 2024-12-10T12:35:55,536 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/46828cc12e2c41dcad1d3bdd2b3ba51d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/46828cc12e2c41dcad1d3bdd2b3ba51d 2024-12-10T12:35:55,536 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/36ba2594a1db4869b02f821d3338f315 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/36ba2594a1db4869b02f821d3338f315 2024-12-10T12:35:55,536 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/f7d644e757924ae7b846a185776cb218 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/f7d644e757924ae7b846a185776cb218 2024-12-10T12:35:55,536 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/afb921e4e35740b69c112bd2a902b137 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/afb921e4e35740b69c112bd2a902b137 2024-12-10T12:35:55,536 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/27e08c1968c84ed7a1b3438c7f77ba21 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/27e08c1968c84ed7a1b3438c7f77ba21 2024-12-10T12:35:55,537 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/eefba4d0edba4588bf614266ee515696 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/eefba4d0edba4588bf614266ee515696 2024-12-10T12:35:55,537 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9c91463a2f304963a6b9eb8c225c9751 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/9c91463a2f304963a6b9eb8c225c9751 2024-12-10T12:35:55,537 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/7243db1f3f5d472899ad50824b562843 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/7243db1f3f5d472899ad50824b562843 2024-12-10T12:35:55,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/1a3b1398f17f4661acf598f426efb8ab, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/9c488a92b1c54ef9bd2f7c8af30e27c3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/4a37e41d17da4934af08bc6b1fd1e716, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aad3d48420d542ce98a4ca17bf5f261f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/f8d6dc06114346bfb911c806b15101d2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dda3e8b2201147c4a265779c97e3d669, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/77c3e1ffb1cc4bcd98fe51d40f9eb6e4, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/7186964619264217bdc1eb08602edb90, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/ca5e4126f8404ea69f79ed0bab6fe34b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/55ee374278dc4cb6b04c79bc99729bf6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/8a357c5946bd4446be0ff784950d638e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/2e2af518f621487b9b344f1d768f48ea, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/499aee31be594624bd2f0b8448172475, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/21eea7ebdd0a4115815b34a2961e28f2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/0bc48d59b66c495ca9bbf60f0b367e7f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/044ac32adde74d02b070d374d66cdbe6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/814fddcbaac0405e9d0b0b61b7433af2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/3b4a9b4526f14b51a7fd363dba61e2fb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dbdc0b2af9f94229873e873aef8d4bf0] to archive 2024-12-10T12:35:55,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:35:55,540 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/9c488a92b1c54ef9bd2f7c8af30e27c3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/9c488a92b1c54ef9bd2f7c8af30e27c3 2024-12-10T12:35:55,540 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/1a3b1398f17f4661acf598f426efb8ab to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/1a3b1398f17f4661acf598f426efb8ab 2024-12-10T12:35:55,540 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/4a37e41d17da4934af08bc6b1fd1e716 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/4a37e41d17da4934af08bc6b1fd1e716 2024-12-10T12:35:55,540 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aad3d48420d542ce98a4ca17bf5f261f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aad3d48420d542ce98a4ca17bf5f261f 2024-12-10T12:35:55,540 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/77c3e1ffb1cc4bcd98fe51d40f9eb6e4 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/77c3e1ffb1cc4bcd98fe51d40f9eb6e4 2024-12-10T12:35:55,540 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dda3e8b2201147c4a265779c97e3d669 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dda3e8b2201147c4a265779c97e3d669 2024-12-10T12:35:55,540 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/7186964619264217bdc1eb08602edb90 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/7186964619264217bdc1eb08602edb90 2024-12-10T12:35:55,540 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/f8d6dc06114346bfb911c806b15101d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/f8d6dc06114346bfb911c806b15101d2 2024-12-10T12:35:55,541 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/ca5e4126f8404ea69f79ed0bab6fe34b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/ca5e4126f8404ea69f79ed0bab6fe34b 2024-12-10T12:35:55,541 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/55ee374278dc4cb6b04c79bc99729bf6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/55ee374278dc4cb6b04c79bc99729bf6 2024-12-10T12:35:55,541 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/8a357c5946bd4446be0ff784950d638e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/8a357c5946bd4446be0ff784950d638e 2024-12-10T12:35:55,541 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/21eea7ebdd0a4115815b34a2961e28f2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/21eea7ebdd0a4115815b34a2961e28f2 2024-12-10T12:35:55,542 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/499aee31be594624bd2f0b8448172475 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/499aee31be594624bd2f0b8448172475 2024-12-10T12:35:55,542 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/0bc48d59b66c495ca9bbf60f0b367e7f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/0bc48d59b66c495ca9bbf60f0b367e7f 2024-12-10T12:35:55,542 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/2e2af518f621487b9b344f1d768f48ea to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/2e2af518f621487b9b344f1d768f48ea 2024-12-10T12:35:55,542 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/044ac32adde74d02b070d374d66cdbe6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/044ac32adde74d02b070d374d66cdbe6 2024-12-10T12:35:55,542 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/814fddcbaac0405e9d0b0b61b7433af2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/814fddcbaac0405e9d0b0b61b7433af2 2024-12-10T12:35:55,542 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/3b4a9b4526f14b51a7fd363dba61e2fb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/3b4a9b4526f14b51a7fd363dba61e2fb 2024-12-10T12:35:55,542 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dbdc0b2af9f94229873e873aef8d4bf0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/dbdc0b2af9f94229873e873aef8d4bf0 2024-12-10T12:35:55,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/7cb21b6ab4524eb7b12c443369657308, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3f9123045b34ae7b0c7708fc58bce59, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5ae05380d175463c990feac12035b7df, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/90666f58541548f492bb4cc2b983baa3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/1233887f37cb4fb598e74ab4c28a2974, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/2a43868c6a1840308c4bb8dd67309ba1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/d838b18f472046058752a52409b5370c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/a2d854b2c02c49e780f2ad0496291a5b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5692e96f973447cc9019412e569d12ac, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/3812502b39f84d7887ba015f2e425dab, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/71f143353477468da2d9645bf17aa614, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/4242c866b1624ee9bb355043c08fea04, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ac29556b95aa48dcae17dbd79009cc4d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/374696d35afc4aaca76f081dfd74d305, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/c01927583a064fadb29c77836e4985fa, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/223ff2340c704e5eaab06fd51f7bb185, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/21dce06a32bb4de682786f2979077dd7, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ec27480385be4843a8feefbd1e281820, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/153bc74088a54d609c4f28b19aa58a60] to archive 2024-12-10T12:35:55,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:35:55,545 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/7cb21b6ab4524eb7b12c443369657308 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/7cb21b6ab4524eb7b12c443369657308 2024-12-10T12:35:55,545 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5ae05380d175463c990feac12035b7df to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5ae05380d175463c990feac12035b7df 2024-12-10T12:35:55,545 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/1233887f37cb4fb598e74ab4c28a2974 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/1233887f37cb4fb598e74ab4c28a2974 2024-12-10T12:35:55,545 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/90666f58541548f492bb4cc2b983baa3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/90666f58541548f492bb4cc2b983baa3 2024-12-10T12:35:55,545 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3f9123045b34ae7b0c7708fc58bce59 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3f9123045b34ae7b0c7708fc58bce59 2024-12-10T12:35:55,545 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/2a43868c6a1840308c4bb8dd67309ba1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/2a43868c6a1840308c4bb8dd67309ba1 2024-12-10T12:35:55,545 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/a2d854b2c02c49e780f2ad0496291a5b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/a2d854b2c02c49e780f2ad0496291a5b 2024-12-10T12:35:55,545 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/d838b18f472046058752a52409b5370c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/d838b18f472046058752a52409b5370c 2024-12-10T12:35:55,546 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/4242c866b1624ee9bb355043c08fea04 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/4242c866b1624ee9bb355043c08fea04 2024-12-10T12:35:55,546 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/3812502b39f84d7887ba015f2e425dab to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/3812502b39f84d7887ba015f2e425dab 2024-12-10T12:35:55,546 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5692e96f973447cc9019412e569d12ac to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/5692e96f973447cc9019412e569d12ac 2024-12-10T12:35:55,546 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/71f143353477468da2d9645bf17aa614 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/71f143353477468da2d9645bf17aa614 2024-12-10T12:35:55,547 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ac29556b95aa48dcae17dbd79009cc4d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ac29556b95aa48dcae17dbd79009cc4d 2024-12-10T12:35:55,547 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/374696d35afc4aaca76f081dfd74d305 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/374696d35afc4aaca76f081dfd74d305 2024-12-10T12:35:55,547 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/223ff2340c704e5eaab06fd51f7bb185 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/223ff2340c704e5eaab06fd51f7bb185 2024-12-10T12:35:55,547 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/c01927583a064fadb29c77836e4985fa to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/c01927583a064fadb29c77836e4985fa 2024-12-10T12:35:55,547 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/21dce06a32bb4de682786f2979077dd7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/21dce06a32bb4de682786f2979077dd7 2024-12-10T12:35:55,547 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/153bc74088a54d609c4f28b19aa58a60 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/153bc74088a54d609c4f28b19aa58a60 2024-12-10T12:35:55,547 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ec27480385be4843a8feefbd1e281820 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ec27480385be4843a8feefbd1e281820 2024-12-10T12:35:55,550 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/recovered.edits/323.seqid, newMaxSeqId=323, maxSeqId=1 2024-12-10T12:35:55,550 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6. 2024-12-10T12:35:55,550 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1635): Region close journal for 19609dde5fb8a6633d70503eaf3922b6: 2024-12-10T12:35:55,551 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] handler.UnassignRegionHandler(170): Closed 19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:55,551 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=144 updating hbase:meta row=19609dde5fb8a6633d70503eaf3922b6, regionState=CLOSED 2024-12-10T12:35:55,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=145, resume processing ppid=144 2024-12-10T12:35:55,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=145, ppid=144, state=SUCCESS; CloseRegionProcedure 19609dde5fb8a6633d70503eaf3922b6, server=fad905551524,38809,1733834007617 in 1.8240 sec 2024-12-10T12:35:55,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=144, resume processing ppid=143 2024-12-10T12:35:55,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=144, ppid=143, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=19609dde5fb8a6633d70503eaf3922b6, UNASSIGN in 1.8280 sec 2024-12-10T12:35:55,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=143, resume processing ppid=142 2024-12-10T12:35:55,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=143, ppid=142, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8320 sec 2024-12-10T12:35:55,555 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834155555"}]},"ts":"1733834155555"} 2024-12-10T12:35:55,556 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T12:35:55,596 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T12:35:55,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=142, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.9220 sec 2024-12-10T12:35:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=142 2024-12-10T12:35:55,785 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 142 completed 2024-12-10T12:35:55,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T12:35:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=146, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:55,790 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=146, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=146 2024-12-10T12:35:55,792 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=146, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:55,795 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:55,798 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/recovered.edits] 2024-12-10T12:35:55,803 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/002012b91b0340169f36a72b10544a8b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/002012b91b0340169f36a72b10544a8b 2024-12-10T12:35:55,803 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0073169fda60446da5cb3b6a9fd7f86e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/0073169fda60446da5cb3b6a9fd7f86e 2024-12-10T12:35:55,803 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/e8f99b602055457a995f442db2a819ec to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/e8f99b602055457a995f442db2a819ec 2024-12-10T12:35:55,803 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/27394bfb2f23415595543d9af3f9c7f1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/A/27394bfb2f23415595543d9af3f9c7f1 2024-12-10T12:35:55,807 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/5c77f8a5a4e6479ba0146d9d07581bf3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/5c77f8a5a4e6479ba0146d9d07581bf3 2024-12-10T12:35:55,807 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aa5fad0ad66a40b9bddb4c14dc1b9847 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/aa5fad0ad66a40b9bddb4c14dc1b9847 2024-12-10T12:35:55,807 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/cfa03c539edd4451b13b0298c47f3dd7 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/cfa03c539edd4451b13b0298c47f3dd7 2024-12-10T12:35:55,807 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/d831b6e26715466abcc0747a483e24a3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/B/d831b6e26715466abcc0747a483e24a3 2024-12-10T12:35:55,810 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/773d87d43eb14d9c9c05a730313e7494 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/773d87d43eb14d9c9c05a730313e7494 2024-12-10T12:35:55,810 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ecbfdd63eb4a4761a3f808332db07b37 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ecbfdd63eb4a4761a3f808332db07b37 2024-12-10T12:35:55,810 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3926379326c4ebfb84167d6c89915a3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/b3926379326c4ebfb84167d6c89915a3 2024-12-10T12:35:55,811 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ed15365bbaf740ad92e8353ee0ce6d12 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/C/ed15365bbaf740ad92e8353ee0ce6d12 2024-12-10T12:35:55,813 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/recovered.edits/323.seqid to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6/recovered.edits/323.seqid 2024-12-10T12:35:55,814 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/19609dde5fb8a6633d70503eaf3922b6 2024-12-10T12:35:55,814 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T12:35:55,816 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=146, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:55,818 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T12:35:55,820 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T12:35:55,821 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=146, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:55,821 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T12:35:55,822 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733834155821"}]},"ts":"9223372036854775807"} 2024-12-10T12:35:55,823 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-12-10T12:35:55,823 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 19609dde5fb8a6633d70503eaf3922b6, NAME => 'TestAcidGuarantees,,1733834130660.19609dde5fb8a6633d70503eaf3922b6.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T12:35:55,824 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T12:35:55,824 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733834155824"}]},"ts":"9223372036854775807"} 2024-12-10T12:35:55,825 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-12-10T12:35:55,830 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=146, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:55,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=146, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-12-10T12:35:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=146 2024-12-10T12:35:55,893 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 146 completed 2024-12-10T12:35:55,907 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testScanAtomicity Thread=245 (was 245), OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=327 (was 295) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9420 (was 9432) 2024-12-10T12:35:55,916 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMobGetAtomicity Thread=245, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=327, ProcessCount=11, AvailableMemoryMB=9420 2024-12-10T12:35:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T12:35:55,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:35:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=147, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:55,918 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=147, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T12:35:55,919 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:55,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 147 2024-12-10T12:35:55,919 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=147, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T12:35:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-10T12:35:55,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742299_1475 (size=963) 2024-12-10T12:35:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-10T12:35:56,113 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T12:35:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-10T12:35:56,329 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3 2024-12-10T12:35:56,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742300_1476 (size=53) 2024-12-10T12:35:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-10T12:35:56,741 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:35:56,741 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing ec290d62df7451631c9eef22904102d2, disabling compactions & flushes 2024-12-10T12:35:56,741 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:56,741 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:56,741 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. after waiting 0 ms 2024-12-10T12:35:56,741 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:56,741 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:56,741 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:35:56,742 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=147, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T12:35:56,743 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733834156742"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733834156742"}]},"ts":"1733834156742"} 2024-12-10T12:35:56,744 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-10T12:35:56,746 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=147, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T12:35:56,746 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834156746"}]},"ts":"1733834156746"} 2024-12-10T12:35:56,747 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T12:35:56,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, ASSIGN}] 2024-12-10T12:35:56,806 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, ASSIGN 2024-12-10T12:35:56,807 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, ASSIGN; state=OFFLINE, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=false 2024-12-10T12:35:56,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=148 updating hbase:meta row=ec290d62df7451631c9eef22904102d2, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:56,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE; OpenRegionProcedure ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-10T12:35:57,114 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:57,121 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:57,121 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7285): Opening region: {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:35:57,122 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:57,122 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:35:57,122 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7327): checking encryption for ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:57,122 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7330): checking classloading for ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:57,124 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:57,126 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:57,127 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec290d62df7451631c9eef22904102d2 columnFamilyName A 2024-12-10T12:35:57,127 DEBUG [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:57,127 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(327): Store=ec290d62df7451631c9eef22904102d2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:57,127 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:57,129 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:57,129 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec290d62df7451631c9eef22904102d2 columnFamilyName B 2024-12-10T12:35:57,129 DEBUG [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:57,130 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(327): Store=ec290d62df7451631c9eef22904102d2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:57,130 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:57,131 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:57,131 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec290d62df7451631c9eef22904102d2 columnFamilyName C 2024-12-10T12:35:57,131 DEBUG [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:57,132 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(327): Store=ec290d62df7451631c9eef22904102d2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:57,132 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:57,133 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:57,133 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:57,134 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:35:57,135 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1085): writing seq id for ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:57,137 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:35:57,137 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1102): Opened ec290d62df7451631c9eef22904102d2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62201589, jitterRate=-0.07312409579753876}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:35:57,138 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1001): Region open journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:35:57,138 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., pid=149, masterSystemTime=1733834157114 2024-12-10T12:35:57,139 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:57,139 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:57,140 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=148 updating hbase:meta row=ec290d62df7451631c9eef22904102d2, regionState=OPEN, openSeqNum=2, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:57,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=149, resume processing ppid=148 2024-12-10T12:35:57,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=149, ppid=148, state=SUCCESS; OpenRegionProcedure ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 in 180 msec 2024-12-10T12:35:57,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=148, resume processing ppid=147 2024-12-10T12:35:57,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=148, ppid=147, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, ASSIGN in 336 msec 2024-12-10T12:35:57,143 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=147, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T12:35:57,143 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834157143"}]},"ts":"1733834157143"} 2024-12-10T12:35:57,144 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T12:35:57,188 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=147, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T12:35:57,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=147, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2710 sec 2024-12-10T12:35:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-10T12:35:58,027 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 147 completed 2024-12-10T12:35:58,029 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d5efb7a to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@644b7e6 2024-12-10T12:35:58,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6094c70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:58,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:58,060 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50450, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:58,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T12:35:58,063 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44006, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T12:35:58,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T12:35:58,067 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$14(2798): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:35:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=150, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T12:35:58,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742301_1477 (size=999) 2024-12-10T12:35:58,484 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.963 2024-12-10T12:35:58,484 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.999 2024-12-10T12:35:58,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T12:35:58,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, REOPEN/MOVE}] 2024-12-10T12:35:58,493 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, REOPEN/MOVE 2024-12-10T12:35:58,494 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=152 updating hbase:meta row=ec290d62df7451631c9eef22904102d2, regionState=CLOSING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:58,495 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T12:35:58,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE; CloseRegionProcedure ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:58,647 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:58,648 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(124): Close ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,648 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-12-10T12:35:58,648 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1681): Closing ec290d62df7451631c9eef22904102d2, disabling compactions & flushes 2024-12-10T12:35:58,648 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:58,648 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:58,648 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. after waiting 0 ms 2024-12-10T12:35:58,648 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:58,656 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T12:35:58,657 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:58,657 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1635): Region close journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:35:58,658 WARN [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionServer(3786): Not adding moved region record: ec290d62df7451631c9eef22904102d2 to self. 2024-12-10T12:35:58,659 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(170): Closed ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,660 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=152 updating hbase:meta row=ec290d62df7451631c9eef22904102d2, regionState=CLOSED 2024-12-10T12:35:58,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=153, resume processing ppid=152 2024-12-10T12:35:58,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=153, ppid=152, state=SUCCESS; CloseRegionProcedure ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 in 166 msec 2024-12-10T12:35:58,665 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, REOPEN/MOVE; state=CLOSED, location=fad905551524,38809,1733834007617; forceNewPlan=false, retain=true 2024-12-10T12:35:58,815 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=152 updating hbase:meta row=ec290d62df7451631c9eef22904102d2, regionState=OPENING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:58,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=154, ppid=152, state=RUNNABLE; OpenRegionProcedure ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617}] 2024-12-10T12:35:58,972 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:58,979 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:58,979 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7285): Opening region: {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:35:58,980 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,980 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:35:58,981 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7327): checking encryption for ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,981 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7330): checking classloading for ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,983 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,984 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:58,984 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec290d62df7451631c9eef22904102d2 columnFamilyName A 2024-12-10T12:35:58,986 DEBUG [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:58,987 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(327): Store=ec290d62df7451631c9eef22904102d2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:58,987 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,988 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:58,988 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec290d62df7451631c9eef22904102d2 columnFamilyName B 2024-12-10T12:35:58,988 DEBUG [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:58,988 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(327): Store=ec290d62df7451631c9eef22904102d2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:58,988 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,989 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2024-12-10T12:35:58,989 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec290d62df7451631c9eef22904102d2 columnFamilyName C 2024-12-10T12:35:58,989 DEBUG [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:58,989 INFO [StoreOpener-ec290d62df7451631c9eef22904102d2-1 {}] regionserver.HStore(327): Store=ec290d62df7451631c9eef22904102d2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:35:58,989 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:58,990 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,991 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,992 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T12:35:58,993 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1085): writing seq id for ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:58,994 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1102): Opened ec290d62df7451631c9eef22904102d2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68293060, jitterRate=0.01764589548110962}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T12:35:58,994 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1001): Region open journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:35:58,995 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., pid=154, masterSystemTime=1733834158972 2024-12-10T12:35:58,996 DEBUG [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:58,996 INFO [RS_OPEN_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:58,996 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=152 updating hbase:meta row=ec290d62df7451631c9eef22904102d2, regionState=OPEN, openSeqNum=5, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:35:58,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=154, resume processing ppid=152 2024-12-10T12:35:58,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=154, ppid=152, state=SUCCESS; OpenRegionProcedure ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 in 179 msec 2024-12-10T12:35:58,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=152, resume processing ppid=151 2024-12-10T12:35:58,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=152, ppid=151, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, REOPEN/MOVE in 506 msec 2024-12-10T12:35:59,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=151, resume processing ppid=150 2024-12-10T12:35:59,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=151, ppid=150, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-12-10T12:35:59,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=150, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 933 msec 2024-12-10T12:35:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=150 2024-12-10T12:35:59,004 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7fc332d8 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c9b5141 2024-12-10T12:35:59,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103dfc6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,040 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17327621 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@11a52cdf 2024-12-10T12:35:59,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e047c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,049 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1584f18a to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d7fe431 2024-12-10T12:35:59,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d631a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,055 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5b914bf4 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@91d72db 2024-12-10T12:35:59,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58971172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,064 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3f6a59e4 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d836f78 2024-12-10T12:35:59,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7fe93b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,072 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x53305d9b to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@11c440f7 2024-12-10T12:35:59,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f1754bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,081 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6bb6288a to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58460ef3 2024-12-10T12:35:59,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9113f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,089 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06556601 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6e8cd1ae 2024-12-10T12:35:59,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb75907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,097 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x458a85fd to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d832d43 2024-12-10T12:35:59,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c1d3a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,106 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x410bf0c8 to 127.0.0.1:64063 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15b6349f 2024-12-10T12:35:59,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503a7d2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:35:59,118 DEBUG [hconnection-0x59aea262-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:35:59,118 DEBUG [hconnection-0x7fe1639a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=155, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees 2024-12-10T12:35:59,119 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50464, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,119 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50480, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,120 DEBUG [hconnection-0x688be6ce-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-10T12:35:59,120 DEBUG [hconnection-0x4e91cdd4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,120 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=155, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:35:59,120 DEBUG [hconnection-0x16f7dd80-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,121 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50496, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,121 DEBUG [hconnection-0x1e835172-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,121 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50482, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,121 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50510, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,121 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=155, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:35:59,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:35:59,122 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50518, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,122 DEBUG [hconnection-0x50630653-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,122 DEBUG [hconnection-0x5f0e126e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,123 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50534, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,123 DEBUG [hconnection-0x1af63d46-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,123 DEBUG [hconnection-0x7bb7ac65-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:35:59,124 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50540, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,125 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50558, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,127 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50546, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:35:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:59,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:35:59,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:35:59,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:59,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:35:59,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:59,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:35:59,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:35:59,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108024cacb3e79479f8661805e436d67fb_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834159123/Put/seqid=0 2024-12-10T12:35:59,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50534 deadline: 1733834219148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834219149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834219150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50482 deadline: 1733834219150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834219150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742302_1478 (size=12154) 2024-12-10T12:35:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-10T12:35:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834219251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50534 deadline: 1733834219251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834219251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50482 deadline: 1733834219252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834219252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,273 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:59,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-12-10T12:35:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:35:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] handler.RSProcedureHandler(58): pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=156 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-10T12:35:59,426 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:59,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-12-10T12:35:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:35:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] handler.RSProcedureHandler(58): pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=156 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834219453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834219453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50534 deadline: 1733834219454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834219454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50482 deadline: 1733834219454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,565 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:35:59,568 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108024cacb3e79479f8661805e436d67fb_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108024cacb3e79479f8661805e436d67fb_ec290d62df7451631c9eef22904102d2 2024-12-10T12:35:59,568 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/f9e0782aa8554ff5bf1045238509d47d, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:35:59,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/f9e0782aa8554ff5bf1045238509d47d is 175, key is test_row_0/A:col10/1733834159123/Put/seqid=0 2024-12-10T12:35:59,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742303_1479 (size=30955) 2024-12-10T12:35:59,578 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:59,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-12-10T12:35:59,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:35:59,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] handler.RSProcedureHandler(58): pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=156 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-10T12:35:59,730 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:59,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-12-10T12:35:59,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:35:59,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] handler.RSProcedureHandler(58): pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=156 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834219756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834219756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50482 deadline: 1733834219756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834219757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:35:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50534 deadline: 1733834219758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:35:59,882 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:35:59,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-12-10T12:35:59,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:35:59,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:35:59,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] handler.RSProcedureHandler(58): pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=156 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:35:59,976 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/f9e0782aa8554ff5bf1045238509d47d 2024-12-10T12:35:59,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/1565979412454c2e9b13fb65efe4d834 is 50, key is test_row_0/B:col10/1733834159123/Put/seqid=0 2024-12-10T12:36:00,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742304_1480 (size=12001) 2024-12-10T12:36:00,037 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:00,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-12-10T12:36:00,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:00,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:00,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:00,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] handler.RSProcedureHandler(58): pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:00,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=156 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:00,189 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:00,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-12-10T12:36:00,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:00,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:00,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:00,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] handler.RSProcedureHandler(58): pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=156 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-10T12:36:00,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:00,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834220261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50534 deadline: 1733834220260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:00,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834220261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:00,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834220263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:00,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50482 deadline: 1733834220274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:00,341 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:00,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-12-10T12:36:00,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:00,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:00,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:00,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] handler.RSProcedureHandler(58): pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:00,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=156 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=156 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:00,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/1565979412454c2e9b13fb65efe4d834 2024-12-10T12:36:00,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/66bf174098744af497e4ab51d9f1b11d is 50, key is test_row_0/C:col10/1733834159123/Put/seqid=0 2024-12-10T12:36:00,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742305_1481 (size=12001) 2024-12-10T12:36:00,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/66bf174098744af497e4ab51d9f1b11d 2024-12-10T12:36:00,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/f9e0782aa8554ff5bf1045238509d47d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/f9e0782aa8554ff5bf1045238509d47d 2024-12-10T12:36:00,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/f9e0782aa8554ff5bf1045238509d47d, entries=150, sequenceid=16, filesize=30.2 K 2024-12-10T12:36:00,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/1565979412454c2e9b13fb65efe4d834 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/1565979412454c2e9b13fb65efe4d834 2024-12-10T12:36:00,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/1565979412454c2e9b13fb65efe4d834, entries=150, sequenceid=16, filesize=11.7 K 2024-12-10T12:36:00,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/66bf174098744af497e4ab51d9f1b11d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/66bf174098744af497e4ab51d9f1b11d 2024-12-10T12:36:00,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/66bf174098744af497e4ab51d9f1b11d, entries=150, sequenceid=16, filesize=11.7 K 2024-12-10T12:36:00,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ec290d62df7451631c9eef22904102d2 in 1318ms, sequenceid=16, compaction requested=false 2024-12-10T12:36:00,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:00,493 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:00,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-12-10T12:36:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:00,494 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-10T12:36:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210853714ad18e844ddb0a5f20914b21531_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834159149/Put/seqid=0 2024-12-10T12:36:00,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742306_1482 (size=12154) 2024-12-10T12:36:00,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:00,504 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210853714ad18e844ddb0a5f20914b21531_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210853714ad18e844ddb0a5f20914b21531_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:00,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/6a53dc0bba54458ab08ee93938a974bb, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:00,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/6a53dc0bba54458ab08ee93938a974bb is 175, key is test_row_0/A:col10/1733834159149/Put/seqid=0 2024-12-10T12:36:00,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742307_1483 (size=30955) 2024-12-10T12:36:00,908 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/6a53dc0bba54458ab08ee93938a974bb 2024-12-10T12:36:00,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/7f8e4061c0924758ad24398f0d8c9afb is 50, key is test_row_0/B:col10/1733834159149/Put/seqid=0 2024-12-10T12:36:00,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742308_1484 (size=12001) 2024-12-10T12:36:01,071 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T12:36:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-10T12:36:01,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:01,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834221269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834221269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50534 deadline: 1733834221271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834221271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50482 deadline: 1733834221280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,317 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/7f8e4061c0924758ad24398f0d8c9afb 2024-12-10T12:36:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/3693782f2c5e4e23b4eac8422da974cb is 50, key is test_row_0/C:col10/1733834159149/Put/seqid=0 2024-12-10T12:36:01,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742309_1485 (size=12001) 2024-12-10T12:36:01,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834221372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834221372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834221373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834221574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834221574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834221575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,726 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/3693782f2c5e4e23b4eac8422da974cb 2024-12-10T12:36:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/6a53dc0bba54458ab08ee93938a974bb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/6a53dc0bba54458ab08ee93938a974bb 2024-12-10T12:36:01,732 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/6a53dc0bba54458ab08ee93938a974bb, entries=150, sequenceid=40, filesize=30.2 K 2024-12-10T12:36:01,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/7f8e4061c0924758ad24398f0d8c9afb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/7f8e4061c0924758ad24398f0d8c9afb 2024-12-10T12:36:01,735 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/7f8e4061c0924758ad24398f0d8c9afb, entries=150, sequenceid=40, filesize=11.7 K 2024-12-10T12:36:01,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/3693782f2c5e4e23b4eac8422da974cb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/3693782f2c5e4e23b4eac8422da974cb 2024-12-10T12:36:01,739 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/3693782f2c5e4e23b4eac8422da974cb, entries=150, sequenceid=40, filesize=11.7 K 2024-12-10T12:36:01,739 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ec290d62df7451631c9eef22904102d2 in 1245ms, sequenceid=40, compaction requested=false 2024-12-10T12:36:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:01,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:01,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=156 2024-12-10T12:36:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=156 2024-12-10T12:36:01,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=156, resume processing ppid=155 2024-12-10T12:36:01,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=156, ppid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6190 sec 2024-12-10T12:36:01,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees in 2.6230 sec 2024-12-10T12:36:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:01,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:36:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:01,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121040fa3ba930854aa387a0342c760ece63_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834161876/Put/seqid=0 2024-12-10T12:36:01,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742310_1486 (size=14594) 2024-12-10T12:36:01,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834221893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834221895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834221895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834221996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:01,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834221997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:02,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834222001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:02,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834222199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:02,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834222200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:02,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834222204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:02,285 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:02,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121040fa3ba930854aa387a0342c760ece63_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121040fa3ba930854aa387a0342c760ece63_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:02,288 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/0243600ab0de4a64a125e5cb309b1b9f, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:02,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/0243600ab0de4a64a125e5cb309b1b9f is 175, key is test_row_0/A:col10/1733834161876/Put/seqid=0 2024-12-10T12:36:02,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742311_1487 (size=39549) 2024-12-10T12:36:02,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:02,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834222501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:02,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834222503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:02,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:02,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834222507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:02,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/0243600ab0de4a64a125e5cb309b1b9f 2024-12-10T12:36:02,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/50c178fa33704f2998a95236284448a6 is 50, key is test_row_0/B:col10/1733834161876/Put/seqid=0 2024-12-10T12:36:02,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742312_1488 (size=12001) 2024-12-10T12:36:03,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834223006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:03,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834223008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:03,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834223011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:03,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/50c178fa33704f2998a95236284448a6 2024-12-10T12:36:03,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/a09a9cede4bc455eb53c9bd18f6aa191 is 50, key is test_row_0/C:col10/1733834161876/Put/seqid=0 2024-12-10T12:36:03,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742313_1489 (size=12001) 2024-12-10T12:36:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-10T12:36:03,224 INFO [Thread-2148 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 155 completed 2024-12-10T12:36:03,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:36:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=157, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees 2024-12-10T12:36:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-10T12:36:03,225 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=157, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:36:03,226 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=157, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:36:03,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:36:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50482 deadline: 1733834223288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50534 deadline: 1733834223288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:03,289 DEBUG [Thread-2142 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4139 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:36:03,289 DEBUG [Thread-2140 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4141 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:36:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-10T12:36:03,377 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:03,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-12-10T12:36:03,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:03,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:03,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:03,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] handler.RSProcedureHandler(58): pid=158 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:03,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=158 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=158 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-10T12:36:03,529 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:03,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-12-10T12:36:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:03,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:03,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:03,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] handler.RSProcedureHandler(58): pid=158 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:03,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=158 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=158 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:03,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/a09a9cede4bc455eb53c9bd18f6aa191 2024-12-10T12:36:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/0243600ab0de4a64a125e5cb309b1b9f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/0243600ab0de4a64a125e5cb309b1b9f 2024-12-10T12:36:03,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/0243600ab0de4a64a125e5cb309b1b9f, entries=200, sequenceid=54, filesize=38.6 K 2024-12-10T12:36:03,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/50c178fa33704f2998a95236284448a6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/50c178fa33704f2998a95236284448a6 2024-12-10T12:36:03,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/50c178fa33704f2998a95236284448a6, entries=150, sequenceid=54, filesize=11.7 K 2024-12-10T12:36:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/a09a9cede4bc455eb53c9bd18f6aa191 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a09a9cede4bc455eb53c9bd18f6aa191 2024-12-10T12:36:03,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a09a9cede4bc455eb53c9bd18f6aa191, entries=150, sequenceid=54, filesize=11.7 K 2024-12-10T12:36:03,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for ec290d62df7451631c9eef22904102d2 in 1678ms, sequenceid=54, compaction requested=true 2024-12-10T12:36:03,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:03,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:36:03,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:03,555 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:03,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:36:03,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:03,555 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:03,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:36:03,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:03,555 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:03,555 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:03,555 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/A is initiating minor compaction (all files) 2024-12-10T12:36:03,555 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/B is initiating minor compaction (all files) 2024-12-10T12:36:03,555 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/A in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:03,555 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/B in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:03,556 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/f9e0782aa8554ff5bf1045238509d47d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/6a53dc0bba54458ab08ee93938a974bb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/0243600ab0de4a64a125e5cb309b1b9f] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=99.1 K 2024-12-10T12:36:03,556 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/1565979412454c2e9b13fb65efe4d834, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/7f8e4061c0924758ad24398f0d8c9afb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/50c178fa33704f2998a95236284448a6] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=35.2 K 2024-12-10T12:36:03,556 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:03,556 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/f9e0782aa8554ff5bf1045238509d47d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/6a53dc0bba54458ab08ee93938a974bb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/0243600ab0de4a64a125e5cb309b1b9f] 2024-12-10T12:36:03,556 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1565979412454c2e9b13fb65efe4d834, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733834159123 2024-12-10T12:36:03,556 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting f9e0782aa8554ff5bf1045238509d47d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733834159123 2024-12-10T12:36:03,556 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 7f8e4061c0924758ad24398f0d8c9afb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733834159147 2024-12-10T12:36:03,556 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6a53dc0bba54458ab08ee93938a974bb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733834159147 2024-12-10T12:36:03,556 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0243600ab0de4a64a125e5cb309b1b9f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733834161265 2024-12-10T12:36:03,556 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 50c178fa33704f2998a95236284448a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733834161270 2024-12-10T12:36:03,560 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:03,561 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:03,562 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210aa12cafe2d4148e78b25d06af1cadac4_ec290d62df7451631c9eef22904102d2 store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:03,562 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/e2dcc0d8420849849763211f5b36994d is 50, key is test_row_0/B:col10/1733834161876/Put/seqid=0 2024-12-10T12:36:03,563 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210aa12cafe2d4148e78b25d06af1cadac4_ec290d62df7451631c9eef22904102d2, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:03,564 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210aa12cafe2d4148e78b25d06af1cadac4_ec290d62df7451631c9eef22904102d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:03,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742314_1490 (size=12104) 2024-12-10T12:36:03,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742315_1491 (size=4469) 2024-12-10T12:36:03,567 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#A#compaction#415 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:03,568 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/cb8bb2bf32f94b0d91791a8ef89e68d3 is 175, key is test_row_0/A:col10/1733834161876/Put/seqid=0 2024-12-10T12:36:03,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742316_1492 (size=31058) 2024-12-10T12:36:03,681 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:03,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-12-10T12:36:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:03,682 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-10T12:36:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:03,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:03,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ea14a234ceb5425aa9559f342892f127_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834161893/Put/seqid=0 2024-12-10T12:36:03,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742317_1493 (size=12154) 2024-12-10T12:36:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-10T12:36:03,969 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/e2dcc0d8420849849763211f5b36994d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e2dcc0d8420849849763211f5b36994d 2024-12-10T12:36:03,972 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/B of ec290d62df7451631c9eef22904102d2 into e2dcc0d8420849849763211f5b36994d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:03,973 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:03,973 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/B, priority=13, startTime=1733834163555; duration=0sec 2024-12-10T12:36:03,973 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:03,973 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:B 2024-12-10T12:36:03,973 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:03,973 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:03,973 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/C is initiating minor compaction (all files) 2024-12-10T12:36:03,973 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/C in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:03,974 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/66bf174098744af497e4ab51d9f1b11d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/3693782f2c5e4e23b4eac8422da974cb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a09a9cede4bc455eb53c9bd18f6aa191] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=35.2 K 2024-12-10T12:36:03,974 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 66bf174098744af497e4ab51d9f1b11d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733834159123 2024-12-10T12:36:03,974 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 3693782f2c5e4e23b4eac8422da974cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733834159147 2024-12-10T12:36:03,974 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting a09a9cede4bc455eb53c9bd18f6aa191, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733834161270 2024-12-10T12:36:03,977 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/cb8bb2bf32f94b0d91791a8ef89e68d3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/cb8bb2bf32f94b0d91791a8ef89e68d3 2024-12-10T12:36:03,984 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/A of ec290d62df7451631c9eef22904102d2 into cb8bb2bf32f94b0d91791a8ef89e68d3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:03,984 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:03,984 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/A, priority=13, startTime=1733834163554; duration=0sec 2024-12-10T12:36:03,984 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:03,984 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:A 2024-12-10T12:36:03,988 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:03,988 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/0a9a681e9d724f7e963dd579a399d471 is 50, key is test_row_0/C:col10/1733834161876/Put/seqid=0 2024-12-10T12:36:03,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742318_1494 (size=12104) 2024-12-10T12:36:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:04,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:04,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834224021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834224022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834224022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:04,092 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ea14a234ceb5425aa9559f342892f127_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ea14a234ceb5425aa9559f342892f127_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:04,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/984ae06e581b44a88b681dbc291d7215, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:04,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/984ae06e581b44a88b681dbc291d7215 is 175, key is test_row_0/A:col10/1733834161893/Put/seqid=0 2024-12-10T12:36:04,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742319_1495 (size=30955) 2024-12-10T12:36:04,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834224123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834224124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834224125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834224325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834224326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834224326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-10T12:36:04,395 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/0a9a681e9d724f7e963dd579a399d471 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0a9a681e9d724f7e963dd579a399d471 2024-12-10T12:36:04,398 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/C of ec290d62df7451631c9eef22904102d2 into 0a9a681e9d724f7e963dd579a399d471(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:04,398 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:04,398 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/C, priority=13, startTime=1733834163555; duration=0sec 2024-12-10T12:36:04,398 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:04,398 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:C 2024-12-10T12:36:04,496 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/984ae06e581b44a88b681dbc291d7215 2024-12-10T12:36:04,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/a7a7ecd54b2840269c2b1de06aa4bf80 is 50, key is test_row_0/B:col10/1733834161893/Put/seqid=0 2024-12-10T12:36:04,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742320_1496 (size=12001) 2024-12-10T12:36:04,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834224627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834224628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834224629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:04,909 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/a7a7ecd54b2840269c2b1de06aa4bf80 2024-12-10T12:36:04,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/4505af0433ae4fc6a7c250edcb326c47 is 50, key is test_row_0/C:col10/1733834161893/Put/seqid=0 2024-12-10T12:36:04,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742321_1497 (size=12001) 2024-12-10T12:36:05,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834225132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:05,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834225134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:05,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834225134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-10T12:36:05,329 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/4505af0433ae4fc6a7c250edcb326c47 2024-12-10T12:36:05,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/984ae06e581b44a88b681dbc291d7215 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/984ae06e581b44a88b681dbc291d7215 2024-12-10T12:36:05,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/984ae06e581b44a88b681dbc291d7215, entries=150, sequenceid=77, filesize=30.2 K 2024-12-10T12:36:05,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/a7a7ecd54b2840269c2b1de06aa4bf80 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/a7a7ecd54b2840269c2b1de06aa4bf80 2024-12-10T12:36:05,337 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/a7a7ecd54b2840269c2b1de06aa4bf80, entries=150, sequenceid=77, filesize=11.7 K 2024-12-10T12:36:05,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/4505af0433ae4fc6a7c250edcb326c47 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/4505af0433ae4fc6a7c250edcb326c47 2024-12-10T12:36:05,341 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/4505af0433ae4fc6a7c250edcb326c47, entries=150, sequenceid=77, filesize=11.7 K 2024-12-10T12:36:05,341 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for ec290d62df7451631c9eef22904102d2 in 1659ms, sequenceid=77, compaction requested=false 2024-12-10T12:36:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=158 2024-12-10T12:36:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=158 2024-12-10T12:36:05,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=158, resume processing ppid=157 2024-12-10T12:36:05,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=158, ppid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1160 sec 2024-12-10T12:36:05,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees in 2.1190 sec 2024-12-10T12:36:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:06,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-10T12:36:06,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:06,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:06,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:06,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:06,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:06,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:06,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100fdf7dbbb5d241f3b5b565629763adeb_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834164021/Put/seqid=0 2024-12-10T12:36:06,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742322_1498 (size=14594) 2024-12-10T12:36:06,153 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:06,160 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100fdf7dbbb5d241f3b5b565629763adeb_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100fdf7dbbb5d241f3b5b565629763adeb_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:06,160 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/5c0ca65388244c99a064c2fdde6fcb66, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:06,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/5c0ca65388244c99a064c2fdde6fcb66 is 175, key is test_row_0/A:col10/1733834164021/Put/seqid=0 2024-12-10T12:36:06,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834226160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834226160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834226160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742323_1499 (size=39549) 2024-12-10T12:36:06,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/5c0ca65388244c99a064c2fdde6fcb66 2024-12-10T12:36:06,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/675726f19df944f9a180d968c1a60085 is 50, key is test_row_0/B:col10/1733834164021/Put/seqid=0 2024-12-10T12:36:06,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742324_1500 (size=12001) 2024-12-10T12:36:06,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834226264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834226264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834226264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834226465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834226466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834226466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/675726f19df944f9a180d968c1a60085 2024-12-10T12:36:06,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/2ae10d0a96b449be88e8121e222a0fc8 is 50, key is test_row_0/C:col10/1733834164021/Put/seqid=0 2024-12-10T12:36:06,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742325_1501 (size=12001) 2024-12-10T12:36:06,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834226768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834226768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834226769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:06,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/2ae10d0a96b449be88e8121e222a0fc8 2024-12-10T12:36:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/5c0ca65388244c99a064c2fdde6fcb66 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/5c0ca65388244c99a064c2fdde6fcb66 2024-12-10T12:36:06,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/5c0ca65388244c99a064c2fdde6fcb66, entries=200, sequenceid=96, filesize=38.6 K 2024-12-10T12:36:06,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/675726f19df944f9a180d968c1a60085 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/675726f19df944f9a180d968c1a60085 2024-12-10T12:36:07,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/675726f19df944f9a180d968c1a60085, entries=150, sequenceid=96, filesize=11.7 K 2024-12-10T12:36:07,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/2ae10d0a96b449be88e8121e222a0fc8 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2ae10d0a96b449be88e8121e222a0fc8 2024-12-10T12:36:07,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2ae10d0a96b449be88e8121e222a0fc8, entries=150, sequenceid=96, filesize=11.7 K 2024-12-10T12:36:07,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for ec290d62df7451631c9eef22904102d2 in 862ms, sequenceid=96, compaction requested=true 2024-12-10T12:36:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:36:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:36:07,004 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:07,004 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:36:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:07,005 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:07,005 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:07,005 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/B is initiating minor compaction (all files) 2024-12-10T12:36:07,005 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/A is initiating minor compaction (all files) 2024-12-10T12:36:07,005 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/A in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,005 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/B in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,005 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e2dcc0d8420849849763211f5b36994d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/a7a7ecd54b2840269c2b1de06aa4bf80, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/675726f19df944f9a180d968c1a60085] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=35.3 K 2024-12-10T12:36:07,005 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/cb8bb2bf32f94b0d91791a8ef89e68d3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/984ae06e581b44a88b681dbc291d7215, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/5c0ca65388244c99a064c2fdde6fcb66] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=99.2 K 2024-12-10T12:36:07,005 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,005 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/cb8bb2bf32f94b0d91791a8ef89e68d3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/984ae06e581b44a88b681dbc291d7215, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/5c0ca65388244c99a064c2fdde6fcb66] 2024-12-10T12:36:07,005 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting e2dcc0d8420849849763211f5b36994d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733834161270 2024-12-10T12:36:07,006 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting cb8bb2bf32f94b0d91791a8ef89e68d3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733834161270 2024-12-10T12:36:07,006 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting a7a7ecd54b2840269c2b1de06aa4bf80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733834161893 2024-12-10T12:36:07,006 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 984ae06e581b44a88b681dbc291d7215, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733834161893 2024-12-10T12:36:07,006 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 675726f19df944f9a180d968c1a60085, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733834164021 2024-12-10T12:36:07,006 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5c0ca65388244c99a064c2fdde6fcb66, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733834164021 2024-12-10T12:36:07,009 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:07,010 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:07,011 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/e863c01e466c4c248131f82d2a5d2a63 is 50, key is test_row_0/B:col10/1733834164021/Put/seqid=0 2024-12-10T12:36:07,011 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412106fb2f401c2694a70ae6e48553464c34f_ec290d62df7451631c9eef22904102d2 store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:07,012 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412106fb2f401c2694a70ae6e48553464c34f_ec290d62df7451631c9eef22904102d2, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:07,012 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106fb2f401c2694a70ae6e48553464c34f_ec290d62df7451631c9eef22904102d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:07,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742326_1502 (size=12207) 2024-12-10T12:36:07,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742327_1503 (size=4469) 2024-12-10T12:36:07,017 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/e863c01e466c4c248131f82d2a5d2a63 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e863c01e466c4c248131f82d2a5d2a63 2024-12-10T12:36:07,021 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/B of ec290d62df7451631c9eef22904102d2 into e863c01e466c4c248131f82d2a5d2a63(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:07,021 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:07,021 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/B, priority=13, startTime=1733834167004; duration=0sec 2024-12-10T12:36:07,021 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:07,021 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:B 2024-12-10T12:36:07,021 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:07,021 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:07,021 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/C is initiating minor compaction (all files) 2024-12-10T12:36:07,021 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/C in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,022 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0a9a681e9d724f7e963dd579a399d471, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/4505af0433ae4fc6a7c250edcb326c47, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2ae10d0a96b449be88e8121e222a0fc8] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=35.3 K 2024-12-10T12:36:07,022 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 0a9a681e9d724f7e963dd579a399d471, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733834161270 2024-12-10T12:36:07,022 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 4505af0433ae4fc6a7c250edcb326c47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733834161893 2024-12-10T12:36:07,022 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2ae10d0a96b449be88e8121e222a0fc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733834164021 2024-12-10T12:36:07,027 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:07,027 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/08f7909b917f46febe2d85093cd3d992 is 50, key is test_row_0/C:col10/1733834164021/Put/seqid=0 2024-12-10T12:36:07,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742328_1504 (size=12207) 2024-12-10T12:36:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:07,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-10T12:36:07,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:07,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:07,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:07,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:07,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:07,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:07,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121019c4b2e92a334a5c98ade0742939aede_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834166160/Put/seqid=0 2024-12-10T12:36:07,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742329_1505 (size=14594) 2024-12-10T12:36:07,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834227282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834227282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834227284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50482 deadline: 1733834227306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,308 DEBUG [Thread-2142 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8158 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:36:07,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50534 deadline: 1733834227316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,319 DEBUG [Thread-2140 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8170 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:36:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-10T12:36:07,329 INFO [Thread-2148 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 157 completed 2024-12-10T12:36:07,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:36:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=159, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=159, table=TestAcidGuarantees 2024-12-10T12:36:07,331 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=159, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=159, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:36:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-12-10T12:36:07,331 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=159, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=159, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:36:07,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:36:07,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834227384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834227385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834227385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,417 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#A#compaction#424 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:07,417 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/1d38d5791d0d4c4288c89da2d2d26d88 is 175, key is test_row_0/A:col10/1733834164021/Put/seqid=0 2024-12-10T12:36:07,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742330_1506 (size=31161) 2024-12-10T12:36:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-12-10T12:36:07,434 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/08f7909b917f46febe2d85093cd3d992 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/08f7909b917f46febe2d85093cd3d992 2024-12-10T12:36:07,437 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/C of ec290d62df7451631c9eef22904102d2 into 08f7909b917f46febe2d85093cd3d992(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:07,437 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:07,437 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/C, priority=13, startTime=1733834167004; duration=0sec 2024-12-10T12:36:07,437 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:07,437 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:C 2024-12-10T12:36:07,482 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:07,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:07,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:07,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834227587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834227587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834227588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-12-10T12:36:07,634 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:07,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:07,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:07,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,682 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:07,685 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121019c4b2e92a334a5c98ade0742939aede_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121019c4b2e92a334a5c98ade0742939aede_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:07,685 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/be46bf2c787b4f1ebacb6b904851b927, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:07,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/be46bf2c787b4f1ebacb6b904851b927 is 175, key is test_row_0/A:col10/1733834166160/Put/seqid=0 2024-12-10T12:36:07,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742331_1507 (size=39549) 2024-12-10T12:36:07,787 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:07,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:07,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:07,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,824 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/1d38d5791d0d4c4288c89da2d2d26d88 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1d38d5791d0d4c4288c89da2d2d26d88 2024-12-10T12:36:07,827 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/A of ec290d62df7451631c9eef22904102d2 into 1d38d5791d0d4c4288c89da2d2d26d88(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:07,827 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:07,827 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/A, priority=13, startTime=1733834167004; duration=0sec 2024-12-10T12:36:07,827 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:07,827 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:A 2024-12-10T12:36:07,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834227891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834227892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834227892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:07,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-12-10T12:36:07,939 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:07,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:07,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:07,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:07,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:07,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,089 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/be46bf2c787b4f1ebacb6b904851b927 2024-12-10T12:36:08,091 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:08,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:08,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:08,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/6353e555a7b6439c92be78b29589b98b is 50, key is test_row_0/B:col10/1733834166160/Put/seqid=0 2024-12-10T12:36:08,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742332_1508 (size=12001) 2024-12-10T12:36:08,244 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:08,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:08,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:08,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834228394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:08,396 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:08,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834228395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:08,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:08,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:08,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834228396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-12-10T12:36:08,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/6353e555a7b6439c92be78b29589b98b 2024-12-10T12:36:08,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/2e10e3c9eefd4e999fc22db2a4cd22e1 is 50, key is test_row_0/C:col10/1733834166160/Put/seqid=0 2024-12-10T12:36:08,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742333_1509 (size=12001) 2024-12-10T12:36:08,548 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:08,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:08,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:08,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,700 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:08,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:08,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:08,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,852 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:08,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:08,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:08,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/2e10e3c9eefd4e999fc22db2a4cd22e1 2024-12-10T12:36:08,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/be46bf2c787b4f1ebacb6b904851b927 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/be46bf2c787b4f1ebacb6b904851b927 2024-12-10T12:36:08,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/be46bf2c787b4f1ebacb6b904851b927, entries=200, sequenceid=120, filesize=38.6 K 2024-12-10T12:36:08,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/6353e555a7b6439c92be78b29589b98b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/6353e555a7b6439c92be78b29589b98b 2024-12-10T12:36:08,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/6353e555a7b6439c92be78b29589b98b, entries=150, sequenceid=120, filesize=11.7 K 2024-12-10T12:36:08,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/2e10e3c9eefd4e999fc22db2a4cd22e1 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2e10e3c9eefd4e999fc22db2a4cd22e1 2024-12-10T12:36:08,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2e10e3c9eefd4e999fc22db2a4cd22e1, entries=150, sequenceid=120, filesize=11.7 K 2024-12-10T12:36:08,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ec290d62df7451631c9eef22904102d2 in 1648ms, sequenceid=120, compaction requested=false 2024-12-10T12:36:08,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:09,005 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:09,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-12-10T12:36:09,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:09,005 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-10T12:36:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:09,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bcd1106e7f0542c6a76fbe83fdd13dd6_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834167281/Put/seqid=0 2024-12-10T12:36:09,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742334_1510 (size=12304) 2024-12-10T12:36:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:09,016 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bcd1106e7f0542c6a76fbe83fdd13dd6_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bcd1106e7f0542c6a76fbe83fdd13dd6_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:09,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7f65b0299c484a4e9875d05954e19f72, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:09,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7f65b0299c484a4e9875d05954e19f72 is 175, key is test_row_0/A:col10/1733834167281/Put/seqid=0 2024-12-10T12:36:09,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742335_1511 (size=31105) 2024-12-10T12:36:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:09,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:09,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834229414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:09,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834229415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:09,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834229415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:09,421 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7f65b0299c484a4e9875d05954e19f72 2024-12-10T12:36:09,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/0d830f0cc86c4695882a90223b0a5503 is 50, key is test_row_0/B:col10/1733834167281/Put/seqid=0 2024-12-10T12:36:09,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742336_1512 (size=12151) 2024-12-10T12:36:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-12-10T12:36:09,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834229516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:09,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834229517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:09,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834229517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:09,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834229719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:09,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834229724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:09,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834229724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:09,831 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/0d830f0cc86c4695882a90223b0a5503 2024-12-10T12:36:09,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/cedcb459738b44f89019da467b640153 is 50, key is test_row_0/C:col10/1733834167281/Put/seqid=0 2024-12-10T12:36:09,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742337_1513 (size=12151) 2024-12-10T12:36:10,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834230026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834230026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834230027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,240 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/cedcb459738b44f89019da467b640153 2024-12-10T12:36:10,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7f65b0299c484a4e9875d05954e19f72 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7f65b0299c484a4e9875d05954e19f72 2024-12-10T12:36:10,245 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7f65b0299c484a4e9875d05954e19f72, entries=150, sequenceid=135, filesize=30.4 K 2024-12-10T12:36:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/0d830f0cc86c4695882a90223b0a5503 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d830f0cc86c4695882a90223b0a5503 2024-12-10T12:36:10,248 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d830f0cc86c4695882a90223b0a5503, entries=150, sequenceid=135, filesize=11.9 K 2024-12-10T12:36:10,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/cedcb459738b44f89019da467b640153 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/cedcb459738b44f89019da467b640153 2024-12-10T12:36:10,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/cedcb459738b44f89019da467b640153, entries=150, sequenceid=135, filesize=11.9 K 2024-12-10T12:36:10,252 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ec290d62df7451631c9eef22904102d2 in 1247ms, sequenceid=135, compaction requested=true 2024-12-10T12:36:10,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:10,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:10,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=160 2024-12-10T12:36:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=160 2024-12-10T12:36:10,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=160, resume processing ppid=159 2024-12-10T12:36:10,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=160, ppid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9220 sec 2024-12-10T12:36:10,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=159, table=TestAcidGuarantees in 2.9240 sec 2024-12-10T12:36:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:10,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-10T12:36:10,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:10,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:10,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:10,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:10,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:10,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:10,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f3da166d929e43ffbc7d148d570b45ba_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834170529/Put/seqid=0 2024-12-10T12:36:10,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834230539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834230541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834230541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742338_1514 (size=17284) 2024-12-10T12:36:10,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:10,551 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f3da166d929e43ffbc7d148d570b45ba_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f3da166d929e43ffbc7d148d570b45ba_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:10,552 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/4fab3b2bbfb5409e84f48edbd6c767cc, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:10,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/4fab3b2bbfb5409e84f48edbd6c767cc is 175, key is test_row_0/A:col10/1733834170529/Put/seqid=0 2024-12-10T12:36:10,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742339_1515 (size=48389) 2024-12-10T12:36:10,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834230642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834230643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834230644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834230845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834230846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834230846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:10,956 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/4fab3b2bbfb5409e84f48edbd6c767cc 2024-12-10T12:36:10,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/0d5bbc90d1ca4793aea7e97e452bdc2b is 50, key is test_row_0/B:col10/1733834170529/Put/seqid=0 2024-12-10T12:36:10,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742340_1516 (size=12151) 2024-12-10T12:36:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834231147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834231148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834231148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:11,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/0d5bbc90d1ca4793aea7e97e452bdc2b 2024-12-10T12:36:11,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/317c16bbbc3440a6b2ebb7ed38066d91 is 50, key is test_row_0/C:col10/1733834170529/Put/seqid=0 2024-12-10T12:36:11,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742341_1517 (size=12151) 2024-12-10T12:36:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-12-10T12:36:11,434 INFO [Thread-2148 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 159 completed 2024-12-10T12:36:11,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:36:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=161, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=161, table=TestAcidGuarantees 2024-12-10T12:36:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-10T12:36:11,436 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=161, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=161, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:36:11,437 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=161, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=161, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:36:11,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:36:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-10T12:36:11,588 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:11,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=162 2024-12-10T12:36:11,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:11,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:11,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:11,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=162}] handler.RSProcedureHandler(58): pid=162 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:11,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=162 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=162 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834231653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834231653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:11,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834231654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-10T12:36:11,740 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:11,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=162 2024-12-10T12:36:11,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:11,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:11,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=162}] handler.RSProcedureHandler(58): pid=162 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=162 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=162 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:11,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/317c16bbbc3440a6b2ebb7ed38066d91 2024-12-10T12:36:11,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/4fab3b2bbfb5409e84f48edbd6c767cc as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/4fab3b2bbfb5409e84f48edbd6c767cc 2024-12-10T12:36:11,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/4fab3b2bbfb5409e84f48edbd6c767cc, entries=250, sequenceid=160, filesize=47.3 K 2024-12-10T12:36:11,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/0d5bbc90d1ca4793aea7e97e452bdc2b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d5bbc90d1ca4793aea7e97e452bdc2b 2024-12-10T12:36:11,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d5bbc90d1ca4793aea7e97e452bdc2b, entries=150, sequenceid=160, filesize=11.9 K 2024-12-10T12:36:11,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/317c16bbbc3440a6b2ebb7ed38066d91 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/317c16bbbc3440a6b2ebb7ed38066d91 2024-12-10T12:36:11,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/317c16bbbc3440a6b2ebb7ed38066d91, entries=150, sequenceid=160, filesize=11.9 K 2024-12-10T12:36:11,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ec290d62df7451631c9eef22904102d2 in 1254ms, sequenceid=160, compaction requested=true 2024-12-10T12:36:11,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:11,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:36:11,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:11,784 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:36:11,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:36:11,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:11,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:36:11,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T12:36:11,785 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:36:11,785 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/A is initiating minor compaction (all files) 2024-12-10T12:36:11,785 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/A in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:11,785 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1d38d5791d0d4c4288c89da2d2d26d88, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/be46bf2c787b4f1ebacb6b904851b927, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7f65b0299c484a4e9875d05954e19f72, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/4fab3b2bbfb5409e84f48edbd6c767cc] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=146.7 K 2024-12-10T12:36:11,785 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:11,785 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1d38d5791d0d4c4288c89da2d2d26d88, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/be46bf2c787b4f1ebacb6b904851b927, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7f65b0299c484a4e9875d05954e19f72, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/4fab3b2bbfb5409e84f48edbd6c767cc] 2024-12-10T12:36:11,785 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 1d38d5791d0d4c4288c89da2d2d26d88, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733834164021 2024-12-10T12:36:11,785 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting be46bf2c787b4f1ebacb6b904851b927, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733834166156 2024-12-10T12:36:11,786 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 7f65b0299c484a4e9875d05954e19f72, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733834167281 2024-12-10T12:36:11,786 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 4fab3b2bbfb5409e84f48edbd6c767cc, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733834169413 2024-12-10T12:36:11,787 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:36:11,788 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:36:11,788 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/B is initiating minor compaction (all files) 2024-12-10T12:36:11,788 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/B in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:11,788 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e863c01e466c4c248131f82d2a5d2a63, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/6353e555a7b6439c92be78b29589b98b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d830f0cc86c4695882a90223b0a5503, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d5bbc90d1ca4793aea7e97e452bdc2b] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=47.4 K 2024-12-10T12:36:11,788 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting e863c01e466c4c248131f82d2a5d2a63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733834164021 2024-12-10T12:36:11,788 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6353e555a7b6439c92be78b29589b98b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733834166160 2024-12-10T12:36:11,788 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0d830f0cc86c4695882a90223b0a5503, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733834167281 2024-12-10T12:36:11,789 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0d5bbc90d1ca4793aea7e97e452bdc2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733834169414 2024-12-10T12:36:11,790 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:11,792 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412108ced218b21ea41fca62d16dba64f0c7d_ec290d62df7451631c9eef22904102d2 store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:11,793 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412108ced218b21ea41fca62d16dba64f0c7d_ec290d62df7451631c9eef22904102d2, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:11,793 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108ced218b21ea41fca62d16dba64f0c7d_ec290d62df7451631c9eef22904102d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:11,794 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#B#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:11,794 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/fad22b418b5542ebacad0f3b26fa59f0 is 50, key is test_row_0/B:col10/1733834170529/Put/seqid=0 2024-12-10T12:36:11,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742342_1518 (size=4469) 2024-12-10T12:36:11,801 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#A#compaction#435 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:11,801 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/3e8552e1a7c942e89871127b8b363816 is 175, key is test_row_0/A:col10/1733834170529/Put/seqid=0 2024-12-10T12:36:11,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742343_1519 (size=12493) 2024-12-10T12:36:11,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742344_1520 (size=31447) 2024-12-10T12:36:11,892 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:11,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=162 2024-12-10T12:36:11,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:11,893 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-10T12:36:11,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:11,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:11,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:11,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:11,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:11,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:11,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a800176d08c34c6298f8edca65e94ea3_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834170540/Put/seqid=0 2024-12-10T12:36:11,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742345_1521 (size=12304) 2024-12-10T12:36:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-10T12:36:12,212 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/fad22b418b5542ebacad0f3b26fa59f0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/fad22b418b5542ebacad0f3b26fa59f0 2024-12-10T12:36:12,215 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/3e8552e1a7c942e89871127b8b363816 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/3e8552e1a7c942e89871127b8b363816 2024-12-10T12:36:12,215 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in ec290d62df7451631c9eef22904102d2/B of ec290d62df7451631c9eef22904102d2 into fad22b418b5542ebacad0f3b26fa59f0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:12,215 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:12,215 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/B, priority=12, startTime=1733834171784; duration=0sec 2024-12-10T12:36:12,215 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:12,215 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:B 2024-12-10T12:36:12,215 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T12:36:12,216 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T12:36:12,216 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/C is initiating minor compaction (all files) 2024-12-10T12:36:12,217 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/C in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:12,217 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/08f7909b917f46febe2d85093cd3d992, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2e10e3c9eefd4e999fc22db2a4cd22e1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/cedcb459738b44f89019da467b640153, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/317c16bbbc3440a6b2ebb7ed38066d91] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=47.4 K 2024-12-10T12:36:12,217 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 08f7909b917f46febe2d85093cd3d992, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733834164021 2024-12-10T12:36:12,217 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2e10e3c9eefd4e999fc22db2a4cd22e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733834166160 2024-12-10T12:36:12,217 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting cedcb459738b44f89019da467b640153, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733834167281 2024-12-10T12:36:12,218 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 317c16bbbc3440a6b2ebb7ed38066d91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733834169414 2024-12-10T12:36:12,218 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in ec290d62df7451631c9eef22904102d2/A of ec290d62df7451631c9eef22904102d2 into 3e8552e1a7c942e89871127b8b363816(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:12,219 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:12,219 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/A, priority=12, startTime=1733834171784; duration=0sec 2024-12-10T12:36:12,219 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:12,219 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:A 2024-12-10T12:36:12,224 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:12,225 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/0f69ea228d74460e9af7a38b72408b06 is 50, key is test_row_0/C:col10/1733834170529/Put/seqid=0 2024-12-10T12:36:12,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742346_1522 (size=12493) 2024-12-10T12:36:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:12,306 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a800176d08c34c6298f8edca65e94ea3_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a800176d08c34c6298f8edca65e94ea3_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:12,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7fc9bdf3df8f477f91a26f93b6c1865c, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:12,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7fc9bdf3df8f477f91a26f93b6c1865c is 175, key is test_row_0/A:col10/1733834170540/Put/seqid=0 2024-12-10T12:36:12,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742347_1523 (size=31105) 2024-12-10T12:36:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-10T12:36:12,631 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/0f69ea228d74460e9af7a38b72408b06 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0f69ea228d74460e9af7a38b72408b06 2024-12-10T12:36:12,634 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in ec290d62df7451631c9eef22904102d2/C of ec290d62df7451631c9eef22904102d2 into 0f69ea228d74460e9af7a38b72408b06(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:12,634 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:12,634 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/C, priority=12, startTime=1733834171784; duration=0sec 2024-12-10T12:36:12,635 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:12,635 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:C 2024-12-10T12:36:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:12,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834232678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:12,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834232679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:12,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834232680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:12,711 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7fc9bdf3df8f477f91a26f93b6c1865c 2024-12-10T12:36:12,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/2cc76d4d131a4d8a8de1b3c62277eedb is 50, key is test_row_0/B:col10/1733834170540/Put/seqid=0 2024-12-10T12:36:12,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742348_1524 (size=12151) 2024-12-10T12:36:12,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834232781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:12,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834232782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:12,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834232782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:12,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834232984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:12,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834232984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:12,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834232984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:13,123 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/2cc76d4d131a4d8a8de1b3c62277eedb 2024-12-10T12:36:13,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/2dc869489c444769abc4a97aebf8f9b2 is 50, key is test_row_0/C:col10/1733834170540/Put/seqid=0 2024-12-10T12:36:13,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742349_1525 (size=12151) 2024-12-10T12:36:13,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834233286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:13,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834233286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:13,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834233288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:13,539 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/2dc869489c444769abc4a97aebf8f9b2 2024-12-10T12:36:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-10T12:36:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7fc9bdf3df8f477f91a26f93b6c1865c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7fc9bdf3df8f477f91a26f93b6c1865c 2024-12-10T12:36:13,565 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7fc9bdf3df8f477f91a26f93b6c1865c, entries=150, sequenceid=171, filesize=30.4 K 2024-12-10T12:36:13,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/2cc76d4d131a4d8a8de1b3c62277eedb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2cc76d4d131a4d8a8de1b3c62277eedb 2024-12-10T12:36:13,568 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2cc76d4d131a4d8a8de1b3c62277eedb, entries=150, sequenceid=171, filesize=11.9 K 2024-12-10T12:36:13,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/2dc869489c444769abc4a97aebf8f9b2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2dc869489c444769abc4a97aebf8f9b2 2024-12-10T12:36:13,572 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2dc869489c444769abc4a97aebf8f9b2, entries=150, sequenceid=171, filesize=11.9 K 2024-12-10T12:36:13,572 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ec290d62df7451631c9eef22904102d2 in 1679ms, sequenceid=171, compaction requested=false 2024-12-10T12:36:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=162 2024-12-10T12:36:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=162 2024-12-10T12:36:13,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=162, resume processing ppid=161 2024-12-10T12:36:13,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=162, ppid=161, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1360 sec 2024-12-10T12:36:13,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=161, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=161, table=TestAcidGuarantees in 2.1390 sec 2024-12-10T12:36:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:13,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-10T12:36:13,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:13,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:13,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:13,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:13,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:13,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:13,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d3e2dce6314b4e7d8a7d1392423bf3c1_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834173791/Put/seqid=0 2024-12-10T12:36:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834233797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:13,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:13,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834233798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834233798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:13,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742350_1526 (size=14794) 2024-12-10T12:36:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834233900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834233901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:13,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834233901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:14,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834234103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:14,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834234103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:14,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834234104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:14,210 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:14,212 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d3e2dce6314b4e7d8a7d1392423bf3c1_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d3e2dce6314b4e7d8a7d1392423bf3c1_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:14,213 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/fcdf2d9d803a43a89081a2129c73264d, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:14,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/fcdf2d9d803a43a89081a2129c73264d is 175, key is test_row_0/A:col10/1733834173791/Put/seqid=0 2024-12-10T12:36:14,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742351_1527 (size=39749) 2024-12-10T12:36:14,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834234404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:14,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834234406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:14,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834234407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:14,616 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/fcdf2d9d803a43a89081a2129c73264d 2024-12-10T12:36:14,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/8ea0ef046e154d529c3040f3eb59dae6 is 50, key is test_row_0/B:col10/1733834173791/Put/seqid=0 2024-12-10T12:36:14,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742352_1528 (size=12151) 2024-12-10T12:36:14,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834234908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:14,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834234909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:14,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834234910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:15,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/8ea0ef046e154d529c3040f3eb59dae6 2024-12-10T12:36:15,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/c09110808f6a40a8b7a7fd9e3a0ddadb is 50, key is test_row_0/C:col10/1733834173791/Put/seqid=0 2024-12-10T12:36:15,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742353_1529 (size=12151) 2024-12-10T12:36:15,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/c09110808f6a40a8b7a7fd9e3a0ddadb 2024-12-10T12:36:15,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/fcdf2d9d803a43a89081a2129c73264d as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/fcdf2d9d803a43a89081a2129c73264d 2024-12-10T12:36:15,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/fcdf2d9d803a43a89081a2129c73264d, entries=200, sequenceid=201, filesize=38.8 K 2024-12-10T12:36:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/8ea0ef046e154d529c3040f3eb59dae6 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/8ea0ef046e154d529c3040f3eb59dae6 2024-12-10T12:36:15,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/8ea0ef046e154d529c3040f3eb59dae6, entries=150, sequenceid=201, filesize=11.9 K 2024-12-10T12:36:15,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/c09110808f6a40a8b7a7fd9e3a0ddadb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/c09110808f6a40a8b7a7fd9e3a0ddadb 2024-12-10T12:36:15,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/c09110808f6a40a8b7a7fd9e3a0ddadb, entries=150, sequenceid=201, filesize=11.9 K 2024-12-10T12:36:15,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for ec290d62df7451631c9eef22904102d2 in 1255ms, sequenceid=201, compaction requested=true 2024-12-10T12:36:15,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:15,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:36:15,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:15,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:36:15,048 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:15,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:15,048 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:15,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:36:15,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/B is initiating minor compaction (all files) 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/A is initiating minor compaction (all files) 2024-12-10T12:36:15,049 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/B in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:15,049 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/A in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:15,049 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/fad22b418b5542ebacad0f3b26fa59f0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2cc76d4d131a4d8a8de1b3c62277eedb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/8ea0ef046e154d529c3040f3eb59dae6] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=35.9 K 2024-12-10T12:36:15,049 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/3e8552e1a7c942e89871127b8b363816, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7fc9bdf3df8f477f91a26f93b6c1865c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/fcdf2d9d803a43a89081a2129c73264d] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=99.9 K 2024-12-10T12:36:15,049 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/3e8552e1a7c942e89871127b8b363816, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7fc9bdf3df8f477f91a26f93b6c1865c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/fcdf2d9d803a43a89081a2129c73264d] 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting fad22b418b5542ebacad0f3b26fa59f0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733834169414 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3e8552e1a7c942e89871127b8b363816, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733834169414 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2cc76d4d131a4d8a8de1b3c62277eedb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733834170538 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7fc9bdf3df8f477f91a26f93b6c1865c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733834170538 2024-12-10T12:36:15,049 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 8ea0ef046e154d529c3040f3eb59dae6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733834172676 2024-12-10T12:36:15,050 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting fcdf2d9d803a43a89081a2129c73264d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733834172676 2024-12-10T12:36:15,064 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:15,064 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#B#compaction#444 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:15,065 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/2777f63d4a5440308653dc0260df0853 is 50, key is test_row_0/B:col10/1733834173791/Put/seqid=0 2024-12-10T12:36:15,066 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210a297d5a098d748bcb88a174a88fadbad_ec290d62df7451631c9eef22904102d2 store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:15,067 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210a297d5a098d748bcb88a174a88fadbad_ec290d62df7451631c9eef22904102d2, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:15,067 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a297d5a098d748bcb88a174a88fadbad_ec290d62df7451631c9eef22904102d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:15,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742354_1530 (size=12595) 2024-12-10T12:36:15,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742355_1531 (size=4469) 2024-12-10T12:36:15,500 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#A#compaction#445 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:15,501 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/2747892e65e0421898c2249579b2574e is 175, key is test_row_0/A:col10/1733834173791/Put/seqid=0 2024-12-10T12:36:15,502 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/2777f63d4a5440308653dc0260df0853 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2777f63d4a5440308653dc0260df0853 2024-12-10T12:36:15,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742356_1532 (size=31549) 2024-12-10T12:36:15,505 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/B of ec290d62df7451631c9eef22904102d2 into 2777f63d4a5440308653dc0260df0853(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:15,505 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:15,505 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/B, priority=13, startTime=1733834175048; duration=0sec 2024-12-10T12:36:15,506 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:15,506 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:B 2024-12-10T12:36:15,506 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:15,506 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:15,506 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/C is initiating minor compaction (all files) 2024-12-10T12:36:15,506 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/C in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:15,507 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0f69ea228d74460e9af7a38b72408b06, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2dc869489c444769abc4a97aebf8f9b2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/c09110808f6a40a8b7a7fd9e3a0ddadb] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=35.9 K 2024-12-10T12:36:15,507 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 0f69ea228d74460e9af7a38b72408b06, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733834169414 2024-12-10T12:36:15,507 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2dc869489c444769abc4a97aebf8f9b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733834170538 2024-12-10T12:36:15,507 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting c09110808f6a40a8b7a7fd9e3a0ddadb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733834172676 2024-12-10T12:36:15,514 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:15,515 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/6abcf2ddf70447189216d67896059150 is 50, key is test_row_0/C:col10/1733834173791/Put/seqid=0 2024-12-10T12:36:15,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742357_1533 (size=12595) 2024-12-10T12:36:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-10T12:36:15,540 INFO [Thread-2148 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 161 completed 2024-12-10T12:36:15,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:36:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=163, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=163, table=TestAcidGuarantees 2024-12-10T12:36:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-10T12:36:15,542 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=163, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=163, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:36:15,543 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=163, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=163, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:36:15,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:36:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-10T12:36:15,698 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:15,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=164 2024-12-10T12:36:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:15,699 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-10T12:36:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:15,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:15,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:15,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f1453a1d442e4d54b2fbdef987ed6a47_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834173797/Put/seqid=0 2024-12-10T12:36:15,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742358_1534 (size=12304) 2024-12-10T12:36:15,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:15,710 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f1453a1d442e4d54b2fbdef987ed6a47_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f1453a1d442e4d54b2fbdef987ed6a47_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:15,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/ed8ea49859914b7bb8c2331f3f1e4ecb, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:15,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/ed8ea49859914b7bb8c2331f3f1e4ecb is 175, key is test_row_0/A:col10/1733834173797/Put/seqid=0 2024-12-10T12:36:15,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742359_1535 (size=31105) 2024-12-10T12:36:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-10T12:36:15,909 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/2747892e65e0421898c2249579b2574e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/2747892e65e0421898c2249579b2574e 2024-12-10T12:36:15,912 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/A of ec290d62df7451631c9eef22904102d2 into 2747892e65e0421898c2249579b2574e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:15,912 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:15,912 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/A, priority=13, startTime=1733834175048; duration=0sec 2024-12-10T12:36:15,912 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:15,912 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:A 2024-12-10T12:36:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:15,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:15,922 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/6abcf2ddf70447189216d67896059150 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/6abcf2ddf70447189216d67896059150 2024-12-10T12:36:15,926 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/C of ec290d62df7451631c9eef22904102d2 into 6abcf2ddf70447189216d67896059150(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:15,926 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:15,926 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/C, priority=13, startTime=1733834175048; duration=0sec 2024-12-10T12:36:15,926 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:15,926 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:C 2024-12-10T12:36:15,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834235936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:15,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834235937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:15,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834235938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834236038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834236039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834236039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,115 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/ed8ea49859914b7bb8c2331f3f1e4ecb 2024-12-10T12:36:16,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/b014ed500a0c4aed937e45e185c11c4a is 50, key is test_row_0/B:col10/1733834173797/Put/seqid=0 2024-12-10T12:36:16,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742360_1536 (size=12151) 2024-12-10T12:36:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-10T12:36:16,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834236240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834236241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834236241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/b014ed500a0c4aed937e45e185c11c4a 2024-12-10T12:36:16,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/a6a65bcf25784e42b5f9def12e52eb1c is 50, key is test_row_0/C:col10/1733834173797/Put/seqid=0 2024-12-10T12:36:16,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742361_1537 (size=12151) 2024-12-10T12:36:16,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834236544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834236544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834236545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-10T12:36:16,938 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/a6a65bcf25784e42b5f9def12e52eb1c 2024-12-10T12:36:16,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/ed8ea49859914b7bb8c2331f3f1e4ecb as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/ed8ea49859914b7bb8c2331f3f1e4ecb 2024-12-10T12:36:16,944 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/ed8ea49859914b7bb8c2331f3f1e4ecb, entries=150, sequenceid=211, filesize=30.4 K 2024-12-10T12:36:16,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/b014ed500a0c4aed937e45e185c11c4a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/b014ed500a0c4aed937e45e185c11c4a 2024-12-10T12:36:16,947 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/b014ed500a0c4aed937e45e185c11c4a, entries=150, sequenceid=211, filesize=11.9 K 2024-12-10T12:36:16,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/a6a65bcf25784e42b5f9def12e52eb1c as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a6a65bcf25784e42b5f9def12e52eb1c 2024-12-10T12:36:16,950 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a6a65bcf25784e42b5f9def12e52eb1c, entries=150, sequenceid=211, filesize=11.9 K 2024-12-10T12:36:16,950 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(3040): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for ec290d62df7451631c9eef22904102d2 in 1251ms, sequenceid=211, compaction requested=false 2024-12-10T12:36:16,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:16,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:16,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=164 2024-12-10T12:36:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=164 2024-12-10T12:36:16,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=164, resume processing ppid=163 2024-12-10T12:36:16,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=164, ppid=163, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-12-10T12:36:16,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=163, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=163, table=TestAcidGuarantees in 1.4110 sec 2024-12-10T12:36:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(8581): Flush requested on ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:17,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-10T12:36:17,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:17,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:17,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:17,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:17,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:17,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:17,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834237051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834237052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834237054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d0c7081464f24560b8c3c204ed79ece9_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834175934/Put/seqid=0 2024-12-10T12:36:17,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742362_1538 (size=12304) 2024-12-10T12:36:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834237155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834237156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834237156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834237358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834237358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834237358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50482 deadline: 1733834237395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,397 DEBUG [Thread-2142 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=8, retries=16, started=18247 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:36:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50534 deadline: 1733834237406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,407 DEBUG [Thread-2140 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=8, retries=16, started=18259 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., hostname=fad905551524,38809,1733834007617, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:36:17,459 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:17,461 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d0c7081464f24560b8c3c204ed79ece9_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d0c7081464f24560b8c3c204ed79ece9_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:17,462 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7078eafcaa7a492aa3e78cccaf5d5c41, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:17,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7078eafcaa7a492aa3e78cccaf5d5c41 is 175, key is test_row_0/A:col10/1733834175934/Put/seqid=0 2024-12-10T12:36:17,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742363_1539 (size=31105) 2024-12-10T12:36:17,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7078eafcaa7a492aa3e78cccaf5d5c41 2024-12-10T12:36:17,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/4d014f20e1b144f8a476c094953a5989 is 50, key is test_row_0/B:col10/1733834175934/Put/seqid=0 2024-12-10T12:36:17,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742364_1540 (size=12151) 2024-12-10T12:36:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-10T12:36:17,645 INFO [Thread-2148 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 163 completed 2024-12-10T12:36:17,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T12:36:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=165, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=165, table=TestAcidGuarantees 2024-12-10T12:36:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-10T12:36:17,647 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=165, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=165, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T12:36:17,648 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=165, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=165, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T12:36:17,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T12:36:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834237661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834237661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834237661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-10T12:36:17,799 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:17,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=166 2024-12-10T12:36:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:17,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] handler.RSProcedureHandler(58): pid=166 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=166 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=166 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:17,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/4d014f20e1b144f8a476c094953a5989 2024-12-10T12:36:17,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/5c5bb9ad80a04dddb697e3ea856fee27 is 50, key is test_row_0/C:col10/1733834175934/Put/seqid=0 2024-12-10T12:36:17,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742365_1541 (size=12151) 2024-12-10T12:36:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-10T12:36:17,951 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:17,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=166 2024-12-10T12:36:17,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:17,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:17,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:17,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] handler.RSProcedureHandler(58): pid=166 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=166 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=166 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:18,103 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:18,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=166 2024-12-10T12:36:18,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:18,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:18,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:18,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] handler.RSProcedureHandler(58): pid=166 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:18,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=166 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=166 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:18,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50464 deadline: 1733834238163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:18,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50546 deadline: 1733834238164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:18,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-10T12:36:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50558 deadline: 1733834238165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 2024-12-10T12:36:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-10T12:36:18,255 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:18,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=166 2024-12-10T12:36:18,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:18,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. as already flushing 2024-12-10T12:36:18,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:18,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] handler.RSProcedureHandler(58): pid=166 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:18,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-2 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=166 java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4114): Remote procedure failed, pid=166 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:36:18,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/5c5bb9ad80a04dddb697e3ea856fee27 2024-12-10T12:36:18,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/7078eafcaa7a492aa3e78cccaf5d5c41 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7078eafcaa7a492aa3e78cccaf5d5c41 2024-12-10T12:36:18,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7078eafcaa7a492aa3e78cccaf5d5c41, entries=150, sequenceid=242, filesize=30.4 K 2024-12-10T12:36:18,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/4d014f20e1b144f8a476c094953a5989 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/4d014f20e1b144f8a476c094953a5989 2024-12-10T12:36:18,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/4d014f20e1b144f8a476c094953a5989, entries=150, sequenceid=242, filesize=11.9 K 2024-12-10T12:36:18,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/5c5bb9ad80a04dddb697e3ea856fee27 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/5c5bb9ad80a04dddb697e3ea856fee27 2024-12-10T12:36:18,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/5c5bb9ad80a04dddb697e3ea856fee27, entries=150, sequenceid=242, filesize=11.9 K 2024-12-10T12:36:18,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for ec290d62df7451631c9eef22904102d2 in 1245ms, sequenceid=242, compaction requested=true 2024-12-10T12:36:18,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:18,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T12:36:18,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:18,294 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:18,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T12:36:18,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:18,294 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:18,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec290d62df7451631c9eef22904102d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T12:36:18,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:18,295 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:18,295 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/A is initiating minor compaction (all files) 2024-12-10T12:36:18,295 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/A in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:18,295 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/2747892e65e0421898c2249579b2574e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/ed8ea49859914b7bb8c2331f3f1e4ecb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7078eafcaa7a492aa3e78cccaf5d5c41] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=91.6 K 2024-12-10T12:36:18,295 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:18,295 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. files: [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/2747892e65e0421898c2249579b2574e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/ed8ea49859914b7bb8c2331f3f1e4ecb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7078eafcaa7a492aa3e78cccaf5d5c41] 2024-12-10T12:36:18,296 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:18,296 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/B is initiating minor compaction (all files) 2024-12-10T12:36:18,296 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2747892e65e0421898c2249579b2574e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733834172676 2024-12-10T12:36:18,296 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/B in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:18,296 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2777f63d4a5440308653dc0260df0853, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/b014ed500a0c4aed937e45e185c11c4a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/4d014f20e1b144f8a476c094953a5989] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=36.0 K 2024-12-10T12:36:18,296 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting ed8ea49859914b7bb8c2331f3f1e4ecb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733834173794 2024-12-10T12:36:18,296 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 2777f63d4a5440308653dc0260df0853, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733834172676 2024-12-10T12:36:18,296 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7078eafcaa7a492aa3e78cccaf5d5c41, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733834175934 2024-12-10T12:36:18,296 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting b014ed500a0c4aed937e45e185c11c4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733834173794 2024-12-10T12:36:18,296 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 4d014f20e1b144f8a476c094953a5989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733834175934 2024-12-10T12:36:18,300 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:18,301 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#B#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:18,302 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121009caf0a4be8c42869e51db1c1e0c59ad_ec290d62df7451631c9eef22904102d2 store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:18,302 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/a17527ede67040c0b00fd47779dd1e53 is 50, key is test_row_0/B:col10/1733834175934/Put/seqid=0 2024-12-10T12:36:18,303 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121009caf0a4be8c42869e51db1c1e0c59ad_ec290d62df7451631c9eef22904102d2, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:18,303 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121009caf0a4be8c42869e51db1c1e0c59ad_ec290d62df7451631c9eef22904102d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:18,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742366_1542 (size=12697) 2024-12-10T12:36:18,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742367_1543 (size=4469) 2024-12-10T12:36:18,407 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:18,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38809 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=166 2024-12-10T12:36:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:18,408 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-10T12:36:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:18,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104c9d1d033b254d1382d6f03372bfb5c4_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834177051/Put/seqid=0 2024-12-10T12:36:18,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742368_1544 (size=12304) 2024-12-10T12:36:18,709 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#A#compaction#453 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:18,709 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/777045740dd1485c92c27bdeea4619de is 175, key is test_row_0/A:col10/1733834175934/Put/seqid=0 2024-12-10T12:36:18,729 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/a17527ede67040c0b00fd47779dd1e53 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/a17527ede67040c0b00fd47779dd1e53 2024-12-10T12:36:18,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742369_1545 (size=31651) 2024-12-10T12:36:18,733 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/777045740dd1485c92c27bdeea4619de as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/777045740dd1485c92c27bdeea4619de 2024-12-10T12:36:18,733 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/B of ec290d62df7451631c9eef22904102d2 into a17527ede67040c0b00fd47779dd1e53(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:18,733 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:18,733 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/B, priority=13, startTime=1733834178294; duration=0sec 2024-12-10T12:36:18,733 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T12:36:18,733 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:B 2024-12-10T12:36:18,733 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T12:36:18,734 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T12:36:18,734 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1540): ec290d62df7451631c9eef22904102d2/C is initiating minor compaction (all files) 2024-12-10T12:36:18,734 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of ec290d62df7451631c9eef22904102d2/C in TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:18,734 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/6abcf2ddf70447189216d67896059150, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a6a65bcf25784e42b5f9def12e52eb1c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/5c5bb9ad80a04dddb697e3ea856fee27] into tmpdir=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp, totalSize=36.0 K 2024-12-10T12:36:18,734 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 6abcf2ddf70447189216d67896059150, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733834172676 2024-12-10T12:36:18,734 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting a6a65bcf25784e42b5f9def12e52eb1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733834173794 2024-12-10T12:36:18,735 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] compactions.Compactor(224): Compacting 5c5bb9ad80a04dddb697e3ea856fee27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733834175934 2024-12-10T12:36:18,736 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/A of ec290d62df7451631c9eef22904102d2 into 777045740dd1485c92c27bdeea4619de(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:18,736 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:18,736 INFO [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/A, priority=13, startTime=1733834178294; duration=0sec 2024-12-10T12:36:18,736 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:18,736 DEBUG [RS:0;fad905551524:38809-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:A 2024-12-10T12:36:18,740 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec290d62df7451631c9eef22904102d2#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T12:36:18,740 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/0578d12801a145c0942a8e5e9cb8273a is 50, key is test_row_0/C:col10/1733834175934/Put/seqid=0 2024-12-10T12:36:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-10T12:36:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742370_1546 (size=12697) 2024-12-10T12:36:18,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:18,818 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104c9d1d033b254d1382d6f03372bfb5c4_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104c9d1d033b254d1382d6f03372bfb5c4_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:18,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/1ce141ff538d426cbd361a43283210c3, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:18,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/1ce141ff538d426cbd361a43283210c3 is 175, key is test_row_0/A:col10/1733834177051/Put/seqid=0 2024-12-10T12:36:18,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742371_1547 (size=31105) 2024-12-10T12:36:19,117 DEBUG [Thread-2157 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x410bf0c8 to 127.0.0.1:64063 2024-12-10T12:36:19,117 DEBUG [Thread-2155 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x458a85fd to 127.0.0.1:64063 2024-12-10T12:36:19,117 DEBUG [Thread-2155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:19,117 DEBUG [Thread-2157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:19,117 DEBUG [Thread-2151 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6bb6288a to 127.0.0.1:64063 2024-12-10T12:36:19,117 DEBUG [Thread-2153 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x06556601 to 127.0.0.1:64063 2024-12-10T12:36:19,117 DEBUG [Thread-2151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:19,117 DEBUG [Thread-2153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:19,118 DEBUG [Thread-2149 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x53305d9b to 127.0.0.1:64063 2024-12-10T12:36:19,118 DEBUG [Thread-2149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:19,160 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/0578d12801a145c0942a8e5e9cb8273a as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0578d12801a145c0942a8e5e9cb8273a 2024-12-10T12:36:19,167 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in ec290d62df7451631c9eef22904102d2/C of ec290d62df7451631c9eef22904102d2 into 0578d12801a145c0942a8e5e9cb8273a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T12:36:19,167 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:19,167 INFO [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2., storeName=ec290d62df7451631c9eef22904102d2/C, priority=13, startTime=1733834178294; duration=0sec 2024-12-10T12:36:19,167 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T12:36:19,167 DEBUG [RS:0;fad905551524:38809-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec290d62df7451631c9eef22904102d2:C 2024-12-10T12:36:19,170 DEBUG [Thread-2146 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3f6a59e4 to 127.0.0.1:64063 2024-12-10T12:36:19,170 DEBUG [Thread-2146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:19,174 DEBUG [Thread-2144 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5b914bf4 to 127.0.0.1:64063 2024-12-10T12:36:19,174 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:19,175 DEBUG [Thread-2138 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7fc332d8 to 127.0.0.1:64063 2024-12-10T12:36:19,175 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:19,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/1ce141ff538d426cbd361a43283210c3 2024-12-10T12:36:19,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/0cb2251ed5624f508a4b3b2a9f4ca1fc is 50, key is test_row_0/B:col10/1733834177051/Put/seqid=0 2024-12-10T12:36:19,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742372_1548 (size=12151) 2024-12-10T12:36:19,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/0cb2251ed5624f508a4b3b2a9f4ca1fc 2024-12-10T12:36:19,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/f5f53d0e725d493b91848490be760b8f is 50, key is test_row_0/C:col10/1733834177051/Put/seqid=0 2024-12-10T12:36:19,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742373_1549 (size=12151) 2024-12-10T12:36:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-10T12:36:20,060 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/f5f53d0e725d493b91848490be760b8f 2024-12-10T12:36:20,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/1ce141ff538d426cbd361a43283210c3 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1ce141ff538d426cbd361a43283210c3 2024-12-10T12:36:20,074 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1ce141ff538d426cbd361a43283210c3, entries=150, sequenceid=249, filesize=30.4 K 2024-12-10T12:36:20,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/0cb2251ed5624f508a4b3b2a9f4ca1fc as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0cb2251ed5624f508a4b3b2a9f4ca1fc 2024-12-10T12:36:20,079 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0cb2251ed5624f508a4b3b2a9f4ca1fc, entries=150, sequenceid=249, filesize=11.9 K 2024-12-10T12:36:20,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/f5f53d0e725d493b91848490be760b8f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/f5f53d0e725d493b91848490be760b8f 2024-12-10T12:36:20,083 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/f5f53d0e725d493b91848490be760b8f, entries=150, sequenceid=249, filesize=11.9 K 2024-12-10T12:36:20,084 INFO [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(3040): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=20.13 KB/20610 for ec290d62df7451631c9eef22904102d2 in 1676ms, sequenceid=249, compaction requested=false 2024-12-10T12:36:20,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(2538): Flush status journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:20,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:20,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fad905551524:0-0 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=166 2024-12-10T12:36:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster(4106): Remote procedure done, pid=166 2024-12-10T12:36:20,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=166, resume processing ppid=165 2024-12-10T12:36:20,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=166, ppid=165, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4370 sec 2024-12-10T12:36:20,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=165, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=165, table=TestAcidGuarantees in 2.4400 sec 2024-12-10T12:36:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-10T12:36:21,754 INFO [Thread-2148 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 165 completed 2024-12-10T12:36:26,113 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T12:36:27,418 DEBUG [Thread-2142 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1584f18a to 127.0.0.1:64063 2024-12-10T12:36:27,418 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:27,430 DEBUG [Thread-2140 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x17327621 to 127.0.0.1:64063 2024-12-10T12:36:27,430 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 9174 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 9347 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 9053 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 9183 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 9343 2024-12-10T12:36:27,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T12:36:27,431 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-10T12:36:27,432 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d5efb7a to 127.0.0.1:64063 2024-12-10T12:36:27,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:27,433 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-12-10T12:36:27,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T12:36:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=167, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T12:36:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-10T12:36:27,439 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834187438"}]},"ts":"1733834187438"} 2024-12-10T12:36:27,440 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T12:36:27,480 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T12:36:27,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T12:36:27,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, UNASSIGN}] 2024-12-10T12:36:27,484 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=169, ppid=168, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, UNASSIGN 2024-12-10T12:36:27,485 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=169 updating hbase:meta row=ec290d62df7451631c9eef22904102d2, regionState=CLOSING, regionLocation=fad905551524,38809,1733834007617 2024-12-10T12:36:27,487 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T12:36:27,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE; CloseRegionProcedure ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617}] 2024-12-10T12:36:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-10T12:36:27,640 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fad905551524,38809,1733834007617 2024-12-10T12:36:27,641 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] handler.UnassignRegionHandler(124): Close ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:27,641 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-12-10T12:36:27,641 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1681): Closing ec290d62df7451631c9eef22904102d2, disabling compactions & flushes 2024-12-10T12:36:27,641 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:27,641 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:27,641 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. after waiting 0 ms 2024-12-10T12:36:27,641 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:27,642 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(2837): Flushing ec290d62df7451631c9eef22904102d2 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-10T12:36:27,642 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=A 2024-12-10T12:36:27,643 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:27,643 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=B 2024-12-10T12:36:27,643 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:27,643 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec290d62df7451631c9eef22904102d2, store=C 2024-12-10T12:36:27,643 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T12:36:27,650 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102731f85bf2204a37a528f5063c56d7fe_ec290d62df7451631c9eef22904102d2 is 50, key is test_row_0/A:col10/1733834179174/Put/seqid=0 2024-12-10T12:36:27,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742374_1550 (size=12454) 2024-12-10T12:36:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-10T12:36:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-10T12:36:28,055 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:36:28,064 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102731f85bf2204a37a528f5063c56d7fe_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102731f85bf2204a37a528f5063c56d7fe_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:28,066 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/41c398a0e2d649e7bf9a298b0e5886c4, store: [table=TestAcidGuarantees family=A region=ec290d62df7451631c9eef22904102d2] 2024-12-10T12:36:28,066 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/41c398a0e2d649e7bf9a298b0e5886c4 is 175, key is test_row_0/A:col10/1733834179174/Put/seqid=0 2024-12-10T12:36:28,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742375_1551 (size=31255) 2024-12-10T12:36:28,472 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/41c398a0e2d649e7bf9a298b0e5886c4 2024-12-10T12:36:28,481 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/222701f1a49b41638278c1cbdc929ba4 is 50, key is test_row_0/B:col10/1733834179174/Put/seqid=0 2024-12-10T12:36:28,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742376_1552 (size=12301) 2024-12-10T12:36:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-10T12:36:28,888 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/222701f1a49b41638278c1cbdc929ba4 2024-12-10T12:36:28,901 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/99537ee5216b4bbaa7af77c011710ca0 is 50, key is test_row_0/C:col10/1733834179174/Put/seqid=0 2024-12-10T12:36:28,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742377_1553 (size=12301) 2024-12-10T12:36:29,306 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/99537ee5216b4bbaa7af77c011710ca0 2024-12-10T12:36:29,332 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/A/41c398a0e2d649e7bf9a298b0e5886c4 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/41c398a0e2d649e7bf9a298b0e5886c4 2024-12-10T12:36:29,341 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/41c398a0e2d649e7bf9a298b0e5886c4, entries=150, sequenceid=260, filesize=30.5 K 2024-12-10T12:36:29,342 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/B/222701f1a49b41638278c1cbdc929ba4 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/222701f1a49b41638278c1cbdc929ba4 2024-12-10T12:36:29,347 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/222701f1a49b41638278c1cbdc929ba4, entries=150, sequenceid=260, filesize=12.0 K 2024-12-10T12:36:29,348 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/.tmp/C/99537ee5216b4bbaa7af77c011710ca0 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/99537ee5216b4bbaa7af77c011710ca0 2024-12-10T12:36:29,351 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/99537ee5216b4bbaa7af77c011710ca0, entries=150, sequenceid=260, filesize=12.0 K 2024-12-10T12:36:29,351 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(3040): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ec290d62df7451631c9eef22904102d2 in 1709ms, sequenceid=260, compaction requested=true 2024-12-10T12:36:29,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/f9e0782aa8554ff5bf1045238509d47d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/6a53dc0bba54458ab08ee93938a974bb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/0243600ab0de4a64a125e5cb309b1b9f, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/cb8bb2bf32f94b0d91791a8ef89e68d3, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/984ae06e581b44a88b681dbc291d7215, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/5c0ca65388244c99a064c2fdde6fcb66, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1d38d5791d0d4c4288c89da2d2d26d88, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/be46bf2c787b4f1ebacb6b904851b927, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7f65b0299c484a4e9875d05954e19f72, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/4fab3b2bbfb5409e84f48edbd6c767cc, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/3e8552e1a7c942e89871127b8b363816, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7fc9bdf3df8f477f91a26f93b6c1865c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/fcdf2d9d803a43a89081a2129c73264d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/2747892e65e0421898c2249579b2574e, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/ed8ea49859914b7bb8c2331f3f1e4ecb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7078eafcaa7a492aa3e78cccaf5d5c41] to archive 2024-12-10T12:36:29,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:36:29,355 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/f9e0782aa8554ff5bf1045238509d47d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/f9e0782aa8554ff5bf1045238509d47d 2024-12-10T12:36:29,355 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/6a53dc0bba54458ab08ee93938a974bb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/6a53dc0bba54458ab08ee93938a974bb 2024-12-10T12:36:29,355 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/cb8bb2bf32f94b0d91791a8ef89e68d3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/cb8bb2bf32f94b0d91791a8ef89e68d3 2024-12-10T12:36:29,355 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/0243600ab0de4a64a125e5cb309b1b9f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/0243600ab0de4a64a125e5cb309b1b9f 2024-12-10T12:36:29,356 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/984ae06e581b44a88b681dbc291d7215 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/984ae06e581b44a88b681dbc291d7215 2024-12-10T12:36:29,356 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/5c0ca65388244c99a064c2fdde6fcb66 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/5c0ca65388244c99a064c2fdde6fcb66 2024-12-10T12:36:29,357 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/be46bf2c787b4f1ebacb6b904851b927 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/be46bf2c787b4f1ebacb6b904851b927 2024-12-10T12:36:29,357 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1d38d5791d0d4c4288c89da2d2d26d88 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1d38d5791d0d4c4288c89da2d2d26d88 2024-12-10T12:36:29,358 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7f65b0299c484a4e9875d05954e19f72 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7f65b0299c484a4e9875d05954e19f72 2024-12-10T12:36:29,358 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/4fab3b2bbfb5409e84f48edbd6c767cc to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/4fab3b2bbfb5409e84f48edbd6c767cc 2024-12-10T12:36:29,358 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7fc9bdf3df8f477f91a26f93b6c1865c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7fc9bdf3df8f477f91a26f93b6c1865c 2024-12-10T12:36:29,358 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/fcdf2d9d803a43a89081a2129c73264d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/fcdf2d9d803a43a89081a2129c73264d 2024-12-10T12:36:29,358 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/3e8552e1a7c942e89871127b8b363816 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/3e8552e1a7c942e89871127b8b363816 2024-12-10T12:36:29,358 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/2747892e65e0421898c2249579b2574e to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/2747892e65e0421898c2249579b2574e 2024-12-10T12:36:29,359 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/ed8ea49859914b7bb8c2331f3f1e4ecb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/ed8ea49859914b7bb8c2331f3f1e4ecb 2024-12-10T12:36:29,359 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7078eafcaa7a492aa3e78cccaf5d5c41 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/7078eafcaa7a492aa3e78cccaf5d5c41 2024-12-10T12:36:29,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/1565979412454c2e9b13fb65efe4d834, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/7f8e4061c0924758ad24398f0d8c9afb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e2dcc0d8420849849763211f5b36994d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/50c178fa33704f2998a95236284448a6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/a7a7ecd54b2840269c2b1de06aa4bf80, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e863c01e466c4c248131f82d2a5d2a63, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/675726f19df944f9a180d968c1a60085, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/6353e555a7b6439c92be78b29589b98b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d830f0cc86c4695882a90223b0a5503, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/fad22b418b5542ebacad0f3b26fa59f0, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d5bbc90d1ca4793aea7e97e452bdc2b, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2cc76d4d131a4d8a8de1b3c62277eedb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2777f63d4a5440308653dc0260df0853, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/8ea0ef046e154d529c3040f3eb59dae6, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/b014ed500a0c4aed937e45e185c11c4a, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/4d014f20e1b144f8a476c094953a5989] to archive 2024-12-10T12:36:29,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:36:29,362 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/50c178fa33704f2998a95236284448a6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/50c178fa33704f2998a95236284448a6 2024-12-10T12:36:29,362 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/1565979412454c2e9b13fb65efe4d834 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/1565979412454c2e9b13fb65efe4d834 2024-12-10T12:36:29,363 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e2dcc0d8420849849763211f5b36994d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e2dcc0d8420849849763211f5b36994d 2024-12-10T12:36:29,363 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/7f8e4061c0924758ad24398f0d8c9afb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/7f8e4061c0924758ad24398f0d8c9afb 2024-12-10T12:36:29,363 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/a7a7ecd54b2840269c2b1de06aa4bf80 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/a7a7ecd54b2840269c2b1de06aa4bf80 2024-12-10T12:36:29,363 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e863c01e466c4c248131f82d2a5d2a63 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/e863c01e466c4c248131f82d2a5d2a63 2024-12-10T12:36:29,363 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/6353e555a7b6439c92be78b29589b98b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/6353e555a7b6439c92be78b29589b98b 2024-12-10T12:36:29,363 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/675726f19df944f9a180d968c1a60085 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/675726f19df944f9a180d968c1a60085 2024-12-10T12:36:29,364 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d830f0cc86c4695882a90223b0a5503 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d830f0cc86c4695882a90223b0a5503 2024-12-10T12:36:29,364 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d5bbc90d1ca4793aea7e97e452bdc2b to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0d5bbc90d1ca4793aea7e97e452bdc2b 2024-12-10T12:36:29,365 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/8ea0ef046e154d529c3040f3eb59dae6 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/8ea0ef046e154d529c3040f3eb59dae6 2024-12-10T12:36:29,365 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2777f63d4a5440308653dc0260df0853 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2777f63d4a5440308653dc0260df0853 2024-12-10T12:36:29,365 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/fad22b418b5542ebacad0f3b26fa59f0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/fad22b418b5542ebacad0f3b26fa59f0 2024-12-10T12:36:29,365 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/b014ed500a0c4aed937e45e185c11c4a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/b014ed500a0c4aed937e45e185c11c4a 2024-12-10T12:36:29,365 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2cc76d4d131a4d8a8de1b3c62277eedb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/2cc76d4d131a4d8a8de1b3c62277eedb 2024-12-10T12:36:29,365 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/4d014f20e1b144f8a476c094953a5989 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/4d014f20e1b144f8a476c094953a5989 2024-12-10T12:36:29,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/66bf174098744af497e4ab51d9f1b11d, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/3693782f2c5e4e23b4eac8422da974cb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0a9a681e9d724f7e963dd579a399d471, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a09a9cede4bc455eb53c9bd18f6aa191, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/4505af0433ae4fc6a7c250edcb326c47, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/08f7909b917f46febe2d85093cd3d992, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2ae10d0a96b449be88e8121e222a0fc8, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2e10e3c9eefd4e999fc22db2a4cd22e1, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/cedcb459738b44f89019da467b640153, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0f69ea228d74460e9af7a38b72408b06, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/317c16bbbc3440a6b2ebb7ed38066d91, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2dc869489c444769abc4a97aebf8f9b2, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/6abcf2ddf70447189216d67896059150, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/c09110808f6a40a8b7a7fd9e3a0ddadb, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a6a65bcf25784e42b5f9def12e52eb1c, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/5c5bb9ad80a04dddb697e3ea856fee27] to archive 2024-12-10T12:36:29,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T12:36:29,370 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/66bf174098744af497e4ab51d9f1b11d to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/66bf174098744af497e4ab51d9f1b11d 2024-12-10T12:36:29,370 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/3693782f2c5e4e23b4eac8422da974cb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/3693782f2c5e4e23b4eac8422da974cb 2024-12-10T12:36:29,370 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/4505af0433ae4fc6a7c250edcb326c47 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/4505af0433ae4fc6a7c250edcb326c47 2024-12-10T12:36:29,370 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0a9a681e9d724f7e963dd579a399d471 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0a9a681e9d724f7e963dd579a399d471 2024-12-10T12:36:29,370 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a09a9cede4bc455eb53c9bd18f6aa191 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a09a9cede4bc455eb53c9bd18f6aa191 2024-12-10T12:36:29,370 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2ae10d0a96b449be88e8121e222a0fc8 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2ae10d0a96b449be88e8121e222a0fc8 2024-12-10T12:36:29,370 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/08f7909b917f46febe2d85093cd3d992 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/08f7909b917f46febe2d85093cd3d992 2024-12-10T12:36:29,371 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2e10e3c9eefd4e999fc22db2a4cd22e1 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2e10e3c9eefd4e999fc22db2a4cd22e1 2024-12-10T12:36:29,371 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/cedcb459738b44f89019da467b640153 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/cedcb459738b44f89019da467b640153 2024-12-10T12:36:29,371 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/317c16bbbc3440a6b2ebb7ed38066d91 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/317c16bbbc3440a6b2ebb7ed38066d91 2024-12-10T12:36:29,372 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2dc869489c444769abc4a97aebf8f9b2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/2dc869489c444769abc4a97aebf8f9b2 2024-12-10T12:36:29,372 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/c09110808f6a40a8b7a7fd9e3a0ddadb to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/c09110808f6a40a8b7a7fd9e3a0ddadb 2024-12-10T12:36:29,372 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0f69ea228d74460e9af7a38b72408b06 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0f69ea228d74460e9af7a38b72408b06 2024-12-10T12:36:29,372 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a6a65bcf25784e42b5f9def12e52eb1c to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/a6a65bcf25784e42b5f9def12e52eb1c 2024-12-10T12:36:29,372 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/6abcf2ddf70447189216d67896059150 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/6abcf2ddf70447189216d67896059150 2024-12-10T12:36:29,372 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/5c5bb9ad80a04dddb697e3ea856fee27 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/5c5bb9ad80a04dddb697e3ea856fee27 2024-12-10T12:36:29,377 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/recovered.edits/263.seqid, newMaxSeqId=263, maxSeqId=4 2024-12-10T12:36:29,377 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2. 2024-12-10T12:36:29,377 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1635): Region close journal for ec290d62df7451631c9eef22904102d2: 2024-12-10T12:36:29,379 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] handler.UnassignRegionHandler(170): Closed ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,379 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=169 updating hbase:meta row=ec290d62df7451631c9eef22904102d2, regionState=CLOSED 2024-12-10T12:36:29,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=170, resume processing ppid=169 2024-12-10T12:36:29,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=170, ppid=169, state=SUCCESS; CloseRegionProcedure ec290d62df7451631c9eef22904102d2, server=fad905551524,38809,1733834007617 in 1.8920 sec 2024-12-10T12:36:29,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=169, resume processing ppid=168 2024-12-10T12:36:29,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=169, ppid=168, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec290d62df7451631c9eef22904102d2, UNASSIGN in 1.8970 sec 2024-12-10T12:36:29,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=168, resume processing ppid=167 2024-12-10T12:36:29,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=168, ppid=167, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9000 sec 2024-12-10T12:36:29,383 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733834189382"}]},"ts":"1733834189382"} 2024-12-10T12:36:29,383 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T12:36:29,396 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T12:36:29,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=167, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.9630 sec 2024-12-10T12:36:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-10T12:36:29,547 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 167 completed 2024-12-10T12:36:29,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T12:36:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] procedure2.ProcedureExecutor(1098): Stored pid=171, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:36:29,551 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=171, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:36:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-10T12:36:29,552 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=171, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:36:29,555 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,561 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C, FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/recovered.edits] 2024-12-10T12:36:29,568 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/777045740dd1485c92c27bdeea4619de to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/777045740dd1485c92c27bdeea4619de 2024-12-10T12:36:29,568 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1ce141ff538d426cbd361a43283210c3 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/1ce141ff538d426cbd361a43283210c3 2024-12-10T12:36:29,568 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/41c398a0e2d649e7bf9a298b0e5886c4 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/A/41c398a0e2d649e7bf9a298b0e5886c4 2024-12-10T12:36:29,572 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/222701f1a49b41638278c1cbdc929ba4 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/222701f1a49b41638278c1cbdc929ba4 2024-12-10T12:36:29,572 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/a17527ede67040c0b00fd47779dd1e53 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/a17527ede67040c0b00fd47779dd1e53 2024-12-10T12:36:29,572 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0cb2251ed5624f508a4b3b2a9f4ca1fc to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/B/0cb2251ed5624f508a4b3b2a9f4ca1fc 2024-12-10T12:36:29,576 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0578d12801a145c0942a8e5e9cb8273a to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/0578d12801a145c0942a8e5e9cb8273a 2024-12-10T12:36:29,576 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/99537ee5216b4bbaa7af77c011710ca0 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/99537ee5216b4bbaa7af77c011710ca0 2024-12-10T12:36:29,576 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/f5f53d0e725d493b91848490be760b8f to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/C/f5f53d0e725d493b91848490be760b8f 2024-12-10T12:36:29,579 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/recovered.edits/263.seqid to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2/recovered.edits/263.seqid 2024-12-10T12:36:29,580 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/default/TestAcidGuarantees/ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,580 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T12:36:29,580 DEBUG [PEWorker-1 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T12:36:29,581 DEBUG [PEWorker-1 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T12:36:29,586 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100fdf7dbbb5d241f3b5b565629763adeb_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100fdf7dbbb5d241f3b5b565629763adeb_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,586 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121019c4b2e92a334a5c98ade0742939aede_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121019c4b2e92a334a5c98ade0742939aede_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,586 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102731f85bf2204a37a528f5063c56d7fe_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102731f85bf2204a37a528f5063c56d7fe_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,587 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104c9d1d033b254d1382d6f03372bfb5c4_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104c9d1d033b254d1382d6f03372bfb5c4_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,587 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108024cacb3e79479f8661805e436d67fb_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108024cacb3e79479f8661805e436d67fb_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,587 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121040fa3ba930854aa387a0342c760ece63_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121040fa3ba930854aa387a0342c760ece63_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,587 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a800176d08c34c6298f8edca65e94ea3_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a800176d08c34c6298f8edca65e94ea3_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,588 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210853714ad18e844ddb0a5f20914b21531_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210853714ad18e844ddb0a5f20914b21531_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,588 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bcd1106e7f0542c6a76fbe83fdd13dd6_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bcd1106e7f0542c6a76fbe83fdd13dd6_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,588 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d0c7081464f24560b8c3c204ed79ece9_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d0c7081464f24560b8c3c204ed79ece9_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,588 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d3e2dce6314b4e7d8a7d1392423bf3c1_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d3e2dce6314b4e7d8a7d1392423bf3c1_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,588 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ea14a234ceb5425aa9559f342892f127_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ea14a234ceb5425aa9559f342892f127_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,588 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f1453a1d442e4d54b2fbdef987ed6a47_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f1453a1d442e4d54b2fbdef987ed6a47_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,588 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f3da166d929e43ffbc7d148d570b45ba_ec290d62df7451631c9eef22904102d2 to hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f3da166d929e43ffbc7d148d570b45ba_ec290d62df7451631c9eef22904102d2 2024-12-10T12:36:29,589 DEBUG [PEWorker-1 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T12:36:29,591 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=171, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:36:29,593 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T12:36:29,594 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T12:36:29,595 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=171, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:36:29,595 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T12:36:29,595 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733834189595"}]},"ts":"9223372036854775807"} 2024-12-10T12:36:29,596 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-12-10T12:36:29,596 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => ec290d62df7451631c9eef22904102d2, NAME => 'TestAcidGuarantees,,1733834155917.ec290d62df7451631c9eef22904102d2.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T12:36:29,596 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T12:36:29,597 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733834189597"}]},"ts":"9223372036854775807"} 2024-12-10T12:36:29,598 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-12-10T12:36:29,605 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=171, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T12:36:29,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=171, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-12-10T12:36:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45051 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-10T12:36:29,654 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 171 completed 2024-12-10T12:36:29,661 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMobGetAtomicity Thread=243 (was 245), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=320 (was 327), ProcessCount=11 (was 11), AvailableMemoryMB=9405 (was 9420) 2024-12-10T12:36:29,661 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-10T12:36:29,661 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-10T12:36:29,661 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x38630296 to 127.0.0.1:64063 2024-12-10T12:36:29,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:29,661 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T12:36:29,661 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=408429447, stopped=false 2024-12-10T12:36:29,662 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=fad905551524,45051,1733834006886 2024-12-10T12:36:29,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T12:36:29,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T12:36:29,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:36:29,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:36:29,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-10T12:36:29,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:29,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:36:29,672 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'fad905551524,38809,1733834007617' ***** 2024-12-10T12:36:29,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:36:29,672 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-10T12:36:29,672 INFO [RS:0;fad905551524:38809 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T12:36:29,672 INFO [RS:0;fad905551524:38809 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T12:36:29,673 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-10T12:36:29,673 INFO [RS:0;fad905551524:38809 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T12:36:29,673 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(3579): Received CLOSE for bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:36:29,673 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1224): stopping server fad905551524,38809,1733834007617 2024-12-10T12:36:29,673 DEBUG [RS:0;fad905551524:38809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:29,673 INFO [RS:0;fad905551524:38809 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T12:36:29,673 INFO [RS:0;fad905551524:38809 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T12:36:29,673 INFO [RS:0;fad905551524:38809 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T12:36:29,673 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-10T12:36:29,673 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-12-10T12:36:29,673 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing bab6b7e921719ae441e5ef7d31c45c53, disabling compactions & flushes 2024-12-10T12:36:29,673 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1603): Online Regions={bab6b7e921719ae441e5ef7d31c45c53=hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53., 1588230740=hbase:meta,,1.1588230740} 2024-12-10T12:36:29,673 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:36:29,673 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:36:29,673 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-10T12:36:29,673 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. after waiting 0 ms 2024-12-10T12:36:29,673 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-10T12:36:29,673 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-10T12:36:29,673 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:36:29,673 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T12:36:29,673 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T12:36:29,674 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing bab6b7e921719ae441e5ef7d31c45c53 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-10T12:36:29,674 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=20.55 KB heapSize=35.87 KB 2024-12-10T12:36:29,674 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:36:29,695 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/namespace/bab6b7e921719ae441e5ef7d31c45c53/.tmp/info/54f7a308850d40888f824c073d2580b2 is 45, key is default/info:d/1733834012901/Put/seqid=0 2024-12-10T12:36:29,697 INFO [regionserver/fad905551524:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:36:29,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742378_1554 (size=5037) 2024-12-10T12:36:29,701 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/.tmp/info/322c40576f81481db2656ce3485e878f is 143, key is hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53./info:regioninfo/1733834012746/Put/seqid=0 2024-12-10T12:36:29,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742379_1555 (size=7725) 2024-12-10T12:36:29,874 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:36:29,931 INFO [regionserver/fad905551524:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-10T12:36:29,931 INFO [regionserver/fad905551524:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-10T12:36:30,074 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, bab6b7e921719ae441e5ef7d31c45c53 2024-12-10T12:36:30,100 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/namespace/bab6b7e921719ae441e5ef7d31c45c53/.tmp/info/54f7a308850d40888f824c073d2580b2 2024-12-10T12:36:30,106 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/.tmp/info/322c40576f81481db2656ce3485e878f 2024-12-10T12:36:30,110 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/namespace/bab6b7e921719ae441e5ef7d31c45c53/.tmp/info/54f7a308850d40888f824c073d2580b2 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/namespace/bab6b7e921719ae441e5ef7d31c45c53/info/54f7a308850d40888f824c073d2580b2 2024-12-10T12:36:30,115 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/namespace/bab6b7e921719ae441e5ef7d31c45c53/info/54f7a308850d40888f824c073d2580b2, entries=2, sequenceid=6, filesize=4.9 K 2024-12-10T12:36:30,115 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for bab6b7e921719ae441e5ef7d31c45c53 in 442ms, sequenceid=6, compaction requested=false 2024-12-10T12:36:30,118 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/namespace/bab6b7e921719ae441e5ef7d31c45c53/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-10T12:36:30,119 INFO [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:36:30,119 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for bab6b7e921719ae441e5ef7d31c45c53: 2024-12-10T12:36:30,119 DEBUG [RS_CLOSE_REGION-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733834011443.bab6b7e921719ae441e5ef7d31c45c53. 2024-12-10T12:36:30,127 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/.tmp/rep_barrier/8b88f31c76994951a98eeae13049c519 is 102, key is TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04./rep_barrier:/1733834038246/DeleteFamily/seqid=0 2024-12-10T12:36:30,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742380_1556 (size=6025) 2024-12-10T12:36:30,275 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-12-10T12:36:30,475 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-12-10T12:36:30,532 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/.tmp/rep_barrier/8b88f31c76994951a98eeae13049c519 2024-12-10T12:36:30,558 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/.tmp/table/81c2ebf856b74a948c0af41b900b1265 is 96, key is TestAcidGuarantees,,1733834013150.646ff6269f37d6ebf1ce7cbee04a0c04./table:/1733834038246/DeleteFamily/seqid=0 2024-12-10T12:36:30,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742381_1557 (size=5942) 2024-12-10T12:36:30,676 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1599): Waiting on 1 regions to close 2024-12-10T12:36:30,676 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T12:36:30,676 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-12-10T12:36:30,876 DEBUG [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-12-10T12:36:30,962 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.08 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/.tmp/table/81c2ebf856b74a948c0af41b900b1265 2024-12-10T12:36:30,965 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/.tmp/info/322c40576f81481db2656ce3485e878f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/info/322c40576f81481db2656ce3485e878f 2024-12-10T12:36:30,967 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/info/322c40576f81481db2656ce3485e878f, entries=22, sequenceid=93, filesize=7.5 K 2024-12-10T12:36:30,968 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/.tmp/rep_barrier/8b88f31c76994951a98eeae13049c519 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/rep_barrier/8b88f31c76994951a98eeae13049c519 2024-12-10T12:36:30,970 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/rep_barrier/8b88f31c76994951a98eeae13049c519, entries=6, sequenceid=93, filesize=5.9 K 2024-12-10T12:36:30,971 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/.tmp/table/81c2ebf856b74a948c0af41b900b1265 as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/table/81c2ebf856b74a948c0af41b900b1265 2024-12-10T12:36:30,974 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/table/81c2ebf856b74a948c0af41b900b1265, entries=9, sequenceid=93, filesize=5.8 K 2024-12-10T12:36:30,974 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~20.55 KB/21040, heapSize ~35.82 KB/36680, currentSize=0 B/0 for 1588230740 in 1300ms, sequenceid=93, compaction requested=false 2024-12-10T12:36:30,978 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/data/hbase/meta/1588230740/recovered.edits/96.seqid, newMaxSeqId=96, maxSeqId=1 2024-12-10T12:36:30,978 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T12:36:30,978 INFO [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-10T12:36:30,978 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-10T12:36:30,978 DEBUG [RS_CLOSE_META-regionserver/fad905551524:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T12:36:31,076 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1250): stopping server fad905551524,38809,1733834007617; all regions closed. 2024-12-10T12:36:31,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741834_1010 (size=26050) 2024-12-10T12:36:31,082 DEBUG [RS:0;fad905551524:38809 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/oldWALs 2024-12-10T12:36:31,082 INFO [RS:0;fad905551524:38809 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL fad905551524%2C38809%2C1733834007617.meta:.meta(num 1733834011122) 2024-12-10T12:36:31,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741832_1008 (size=13068376) 2024-12-10T12:36:31,086 DEBUG [RS:0;fad905551524:38809 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/oldWALs 2024-12-10T12:36:31,086 INFO [RS:0;fad905551524:38809 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL fad905551524%2C38809%2C1733834007617:(num 1733834010088) 2024-12-10T12:36:31,086 DEBUG [RS:0;fad905551524:38809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:31,086 INFO [RS:0;fad905551524:38809 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:36:31,086 INFO [RS:0;fad905551524:38809 {}] hbase.ChoreService(370): Chore service for: regionserver/fad905551524:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-10T12:36:31,087 INFO [regionserver/fad905551524:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-10T12:36:31,087 INFO [RS:0;fad905551524:38809 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38809 2024-12-10T12:36:31,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:36:31,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fad905551524,38809,1733834007617 2024-12-10T12:36:31,128 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fad905551524,38809,1733834007617] 2024-12-10T12:36:31,129 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing fad905551524,38809,1733834007617; numProcessing=1 2024-12-10T12:36:31,146 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/fad905551524,38809,1733834007617 already deleted, retry=false 2024-12-10T12:36:31,146 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; fad905551524,38809,1733834007617 expired; onlineServers=0 2024-12-10T12:36:31,147 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'fad905551524,45051,1733834006886' ***** 2024-12-10T12:36:31,147 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T12:36:31,147 DEBUG [M:0;fad905551524:45051 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5945a06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fad905551524/172.17.0.2:0 2024-12-10T12:36:31,147 INFO [M:0;fad905551524:45051 {}] regionserver.HRegionServer(1224): stopping server fad905551524,45051,1733834006886 2024-12-10T12:36:31,147 INFO [M:0;fad905551524:45051 {}] regionserver.HRegionServer(1250): stopping server fad905551524,45051,1733834006886; all regions closed. 2024-12-10T12:36:31,147 DEBUG [M:0;fad905551524:45051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:36:31,147 DEBUG [M:0;fad905551524:45051 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T12:36:31,147 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T12:36:31,147 DEBUG [M:0;fad905551524:45051 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T12:36:31,147 DEBUG [master/fad905551524:0:becomeActiveMaster-HFileCleaner.large.0-1733834009824 {}] cleaner.HFileCleaner(306): Exit Thread[master/fad905551524:0:becomeActiveMaster-HFileCleaner.large.0-1733834009824,5,FailOnTimeoutGroup] 2024-12-10T12:36:31,147 DEBUG [master/fad905551524:0:becomeActiveMaster-HFileCleaner.small.0-1733834009824 {}] cleaner.HFileCleaner(306): Exit Thread[master/fad905551524:0:becomeActiveMaster-HFileCleaner.small.0-1733834009824,5,FailOnTimeoutGroup] 2024-12-10T12:36:31,147 INFO [M:0;fad905551524:45051 {}] hbase.ChoreService(370): Chore service for: master/fad905551524:0 had [] on shutdown 2024-12-10T12:36:31,147 DEBUG [M:0;fad905551524:45051 {}] master.HMaster(1733): Stopping service threads 2024-12-10T12:36:31,147 INFO [M:0;fad905551524:45051 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T12:36:31,148 INFO [M:0;fad905551524:45051 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T12:36:31,148 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T12:36:31,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T12:36:31,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:36:31,155 DEBUG [M:0;fad905551524:45051 {}] zookeeper.ZKUtil(347): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T12:36:31,155 WARN [M:0;fad905551524:45051 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T12:36:31,155 INFO [M:0;fad905551524:45051 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-10T12:36:31,155 INFO [M:0;fad905551524:45051 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T12:36:31,155 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T12:36:31,155 DEBUG [M:0;fad905551524:45051 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T12:36:31,155 INFO [M:0;fad905551524:45051 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:36:31,155 DEBUG [M:0;fad905551524:45051 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:36:31,155 DEBUG [M:0;fad905551524:45051 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T12:36:31,155 DEBUG [M:0;fad905551524:45051 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:36:31,156 INFO [M:0;fad905551524:45051 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=719.18 KB heapSize=880.28 KB 2024-12-10T12:36:31,172 DEBUG [M:0;fad905551524:45051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9f31bc0a724b4aeaa909753ae762126b is 82, key is hbase:meta,,1/info:regioninfo/1733834011233/Put/seqid=0 2024-12-10T12:36:31,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742382_1558 (size=5672) 2024-12-10T12:36:31,238 INFO [RS:0;fad905551524:38809 {}] regionserver.HRegionServer(1307): Exiting; stopping=fad905551524,38809,1733834007617; zookeeper connection closed. 2024-12-10T12:36:31,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:36:31,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38809-0x1000fee7b100001, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:36:31,239 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3da9195d {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3da9195d 2024-12-10T12:36:31,239 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-10T12:36:31,576 INFO [M:0;fad905551524:45051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1972 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9f31bc0a724b4aeaa909753ae762126b 2024-12-10T12:36:31,604 DEBUG [M:0;fad905551524:45051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ce1ec3901f5474385665e2017c9d71e is 2285, key is \x00\x00\x00\x00\x00\x00\x00\x96/proc:d/1733834159002/Put/seqid=0 2024-12-10T12:36:31,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742383_1559 (size=43731) 2024-12-10T12:36:32,008 INFO [M:0;fad905551524:45051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=718.63 KB at sequenceid=1972 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ce1ec3901f5474385665e2017c9d71e 2024-12-10T12:36:32,016 INFO [M:0;fad905551524:45051 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0ce1ec3901f5474385665e2017c9d71e 2024-12-10T12:36:32,038 DEBUG [M:0;fad905551524:45051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97e6761d9088464384f8bc29e5b2278f is 69, key is fad905551524,38809,1733834007617/rs:state/1733834009853/Put/seqid=0 2024-12-10T12:36:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073742384_1560 (size=5156) 2024-12-10T12:36:32,443 INFO [M:0;fad905551524:45051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1972 (bloomFilter=true), to=hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97e6761d9088464384f8bc29e5b2278f 2024-12-10T12:36:32,450 DEBUG [M:0;fad905551524:45051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9f31bc0a724b4aeaa909753ae762126b as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9f31bc0a724b4aeaa909753ae762126b 2024-12-10T12:36:32,455 INFO [M:0;fad905551524:45051 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9f31bc0a724b4aeaa909753ae762126b, entries=8, sequenceid=1972, filesize=5.5 K 2024-12-10T12:36:32,456 DEBUG [M:0;fad905551524:45051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ce1ec3901f5474385665e2017c9d71e as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0ce1ec3901f5474385665e2017c9d71e 2024-12-10T12:36:32,460 INFO [M:0;fad905551524:45051 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0ce1ec3901f5474385665e2017c9d71e 2024-12-10T12:36:32,460 INFO [M:0;fad905551524:45051 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0ce1ec3901f5474385665e2017c9d71e, entries=171, sequenceid=1972, filesize=42.7 K 2024-12-10T12:36:32,460 DEBUG [M:0;fad905551524:45051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97e6761d9088464384f8bc29e5b2278f as hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97e6761d9088464384f8bc29e5b2278f 2024-12-10T12:36:32,463 INFO [M:0;fad905551524:45051 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39921/user/jenkins/test-data/3f1feeb9-470e-649f-609f-656afd2fa8a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97e6761d9088464384f8bc29e5b2278f, entries=1, sequenceid=1972, filesize=5.0 K 2024-12-10T12:36:32,464 INFO [M:0;fad905551524:45051 {}] regionserver.HRegion(3040): Finished flush of dataSize ~719.18 KB/736442, heapSize ~879.98 KB/901104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=1972, compaction requested=false 2024-12-10T12:36:32,465 INFO [M:0;fad905551524:45051 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:36:32,465 DEBUG [M:0;fad905551524:45051 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-10T12:36:32,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741830_1006 (size=865829) 2024-12-10T12:36:32,468 INFO [M:0;fad905551524:45051 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-10T12:36:32,468 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-10T12:36:32,468 INFO [M:0;fad905551524:45051 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45051 2024-12-10T12:36:32,511 DEBUG [M:0;fad905551524:45051 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/fad905551524,45051,1733834006886 already deleted, retry=false 2024-12-10T12:36:32,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:36:32,622 INFO [M:0;fad905551524:45051 {}] regionserver.HRegionServer(1307): Exiting; stopping=fad905551524,45051,1733834006886; zookeeper connection closed. 2024-12-10T12:36:32,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45051-0x1000fee7b100000, quorum=127.0.0.1:64063, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:36:32,628 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10ba49e9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T12:36:32,632 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@661c2e9c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T12:36:32,632 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T12:36:32,632 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ca71a25{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T12:36:32,632 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@134e7cc5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/hadoop.log.dir/,STOPPED} 2024-12-10T12:36:32,636 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T12:36:32,636 WARN [BP-2145257419-172.17.0.2-1733834003915 heartbeating to localhost/127.0.0.1:39921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T12:36:32,636 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T12:36:32,636 WARN [BP-2145257419-172.17.0.2-1733834003915 heartbeating to localhost/127.0.0.1:39921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2145257419-172.17.0.2-1733834003915 (Datanode Uuid 34eaf696-2436-4d2b-85ae-880ce4bf3216) service to localhost/127.0.0.1:39921 2024-12-10T12:36:32,638 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/cluster_c0fdc2cf-aabc-c2dd-a875-8dc8dd174cbc/dfs/data/data1/current/BP-2145257419-172.17.0.2-1733834003915 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T12:36:32,638 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/cluster_c0fdc2cf-aabc-c2dd-a875-8dc8dd174cbc/dfs/data/data2/current/BP-2145257419-172.17.0.2-1733834003915 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T12:36:32,639 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T12:36:32,644 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b03fcff{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T12:36:32,645 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@e0a3ea0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T12:36:32,645 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T12:36:32,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62a9beb2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T12:36:32,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8167a4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/015446fb-7b18-71f5-e989-efcd2b6007d5/hadoop.log.dir/,STOPPED} 2024-12-10T12:36:32,661 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-10T12:36:32,771 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down